Compare commits
51 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5efd9af32 | ||
|
|
9f1e27d011 | ||
|
|
ebc57702dc | ||
|
|
3aa30bc1e2 | ||
|
|
f97fb48e51 | ||
|
|
4b15894603 | ||
|
|
31a14a0352 | ||
|
|
eecbe91dbd | ||
|
|
9dd3947cb7 | ||
|
|
c151f4f76e | ||
|
|
b6755497e4 | ||
|
|
412e24b1e3 | ||
|
|
0a5785fa1a | ||
|
|
7ebd73c9cf | ||
|
|
95cd4028d7 | ||
|
|
eb01c2107c | ||
|
|
42cd4fa0ae | ||
|
|
6d96fd2bf8 | ||
|
|
ff1d2317a1 | ||
|
|
a47fa41278 | ||
|
|
9654e56846 | ||
|
|
43094640a8 | ||
|
|
fa234d2d78 | ||
|
|
7cb0d6e382 | ||
|
|
d79f8d2664 | ||
|
|
4d37e890ab | ||
|
|
7589f20a18 | ||
|
|
702ffabc1a | ||
|
|
9387de1cd9 | ||
|
|
facd683d45 | ||
|
|
8324357edb | ||
|
|
67af7c7dc5 | ||
|
|
6f5b52f3ad | ||
|
|
89d239c360 | ||
|
|
dd5c5b29d9 | ||
|
|
0cb2f48645 | ||
|
|
3f294b8099 | ||
|
|
11fd18e76a | ||
|
|
c839f3abc9 | ||
|
|
28eddc5d7f | ||
|
|
0ca9c5ae02 | ||
|
|
a7d0037edd | ||
|
|
f339a2e2cf | ||
|
|
da8fd13973 | ||
|
|
7deffc64af | ||
|
|
d1dad7fd3b | ||
|
|
4f79c5cba2 | ||
|
|
28e885f6c7 | ||
|
|
1d99fc95f7 | ||
|
|
28f0a167e6 | ||
|
|
0d2b21c789 |
38
.github/workflows/docker.yml
vendored
@@ -57,3 +57,41 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
# Test container health with docker compose
|
||||
- name: Test container with docker compose
|
||||
run: |
|
||||
echo "Starting container with docker compose..."
|
||||
docker compose up --build -d
|
||||
echo "Waiting for container to be ready (60 seconds for start_period)..."
|
||||
sleep 60
|
||||
|
||||
echo "Monitoring container health for 30 seconds..."
|
||||
SECONDS_ELAPSED=0
|
||||
HEALTH_CHECK_INTERVAL=5
|
||||
TOTAL_DURATION=30
|
||||
|
||||
while [ $SECONDS_ELAPSED -lt $TOTAL_DURATION ]; do
|
||||
HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' fredy 2>/dev/null || echo "not_found")
|
||||
CONTAINER_STATUS=$(docker inspect --format='{{.State.Status}}' fredy 2>/dev/null || echo "not_found")
|
||||
echo "[$SECONDS_ELAPSED/$TOTAL_DURATION sec] Container: $CONTAINER_STATUS, Health: $HEALTH_STATUS"
|
||||
|
||||
# Check if container is not running or unhealthy
|
||||
if [ "$CONTAINER_STATUS" != "running" ]; then
|
||||
echo "Container stopped running! Status: $CONTAINER_STATUS"
|
||||
docker compose logs fredy
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$HEALTH_STATUS" = "unhealthy" ]; then
|
||||
echo "Container is unhealthy!"
|
||||
docker compose logs fredy
|
||||
docker inspect --format='{{json .State.Health}}' fredy | jq
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep $HEALTH_CHECK_INTERVAL
|
||||
SECONDS_ELAPSED=$((SECONDS_ELAPSED + HEALTH_CHECK_INTERVAL))
|
||||
done
|
||||
|
||||
docker compose down
|
||||
|
||||
1
.gitignore
vendored
@@ -5,3 +5,4 @@ db/*.db*
|
||||
npm-debug.log
|
||||
.DS_Store
|
||||
.idea
|
||||
.vscode
|
||||
|
||||
@@ -2,9 +2,10 @@ FROM node:22-slim
|
||||
|
||||
WORKDIR /fredy
|
||||
|
||||
# Install Chromium without extra recommended packages and clean apt cache
|
||||
# Install Chromium and curl without extra recommended packages and clean apt cache
|
||||
# curl is needed for the health check
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends chromium \
|
||||
&& apt-get install -y --no-install-recommends chromium curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
@@ -30,6 +31,8 @@ RUN mkdir -p /db /conf \
|
||||
&& ln -s /conf /fredy/conf
|
||||
|
||||
EXPOSE 9998
|
||||
VOLUME /db
|
||||
VOLUME /conf
|
||||
|
||||
# Start application using PM2 runtime
|
||||
CMD ["pm2-runtime", "index.js"]
|
||||
|
||||
30
README.md
@@ -9,10 +9,18 @@
|
||||
</a>
|
||||
</p>
|
||||
|
||||

|
||||
[](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
|
||||

|
||||

|
||||
<p align="center">
|
||||
<a href="https://fredy.orange-coding.net/" target="_blank">Website</a> |
|
||||
<a href="https://demo-fredy.orange-coding.net/" target="_blank">Demo</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg" alt="Tests" />
|
||||
<img src="https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg" alt="Docker" />
|
||||
<img src="https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg" alt="Source" />
|
||||
<img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fghcr-badge.elias.eu.org%2Fapi%2Forangecoding%2Ffredy%2Ffredy&query=%24.downloadCount&label=Docker%20Pulls" alt="Docker Pulls" />
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
# Fredy 🏡 – Your Self-Hosted Real Estate Finder for Germany
|
||||
@@ -21,15 +29,13 @@ Finding an apartment or house in Germany can be stressful and
|
||||
time-consuming.\
|
||||
**Fredy** makes it easier: it automatically scrapes **ImmoScout24,
|
||||
Immowelt, Immonet, eBay Kleinanzeigen, and WG-Gesucht** and notifies you
|
||||
instantly via **Slack, Telegram, Email, ntfy, and more** when new
|
||||
instantly via **Slack, Telegram, Email, ntfy, discord and more** when new
|
||||
listings appear.
|
||||
|
||||
With a modern architecture, Fredy provides a **clean Web UI**, removes
|
||||
duplicates across platforms, and stores results so you never see the
|
||||
same listing twice.
|
||||
|
||||
|
||||
|
||||
------------------------------------------------------------------------
|
||||
|
||||
## ✨ Key Features
|
||||
@@ -37,7 +43,7 @@ same listing twice.
|
||||
- 🏠 Scrapes **ImmoScout24, Immowelt, Immonet, eBay Kleinanzeigen,
|
||||
WG-Gesucht**
|
||||
- ⚡ Instant notifications: Slack, Telegram, Email (SendGrid,
|
||||
Mailjet), ntfy
|
||||
Mailjet), ntfy, discord
|
||||
- 🔎 Uses the **ImmoScout Mobile API** (reverse engineered)
|
||||
- 🌍 Runs anywhere: Docker, Node.js, self-hosted
|
||||
- 🖥️ Intuitive **Web UI** to manage searches
|
||||
@@ -109,9 +115,9 @@ yarn run start:frontend # in another terminal
|
||||
|
||||
## 📸 Screenshots
|
||||
|
||||
| Job Configuration | Job Analytics | Job Overview |
|
||||
|-------------------|--------------|--------------|
|
||||
|  |  |  |
|
||||
| Fredy Main Overview | Job Configuration | Found Listings |
|
||||
|--------------------------------------------------|-----------------------------------------------------------------------|-----------------------------------------------------------------------------|
|
||||
|  |  |  |
|
||||
|
||||
------------------------------------------------------------------------
|
||||
|
||||
@@ -131,7 +137,7 @@ picks up the newest listings first.
|
||||
### Adapter 📡
|
||||
|
||||
An **adapter** is the channel through which Fredy notifies you (Slack,
|
||||
Telegram, Email, ntfy, ...).\
|
||||
Telegram, Email, ntfy, discord ...).\
|
||||
Each adapter has its own configuration (e.g. API keys, webhook URLs).\
|
||||
You can use multiple adapters at once --- Fredy will send new listings
|
||||
through all of them.
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null,"sqlitepath":"/db"}
|
||||
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}
|
||||
@@ -1,199 +0,0 @@
|
||||
/**
|
||||
* Migration Runner for better-sqlite3
|
||||
* I know there are external libs out there, but
|
||||
* a) most of them are pretty bloated
|
||||
* b) I wanted to have something that fit's this limited use-case
|
||||
* c) I was searching for justifications anyway to build a migration system on my own. Don't judge me ;)
|
||||
*
|
||||
* Executes all migration files in db/migrations/sql in natural order.
|
||||
* Each migration runs in its own transaction. If a migration fails, only that
|
||||
* migration is rolled back and the process stops with a non-zero exit code.
|
||||
* Already applied migrations are skipped using the schema_migrations table.
|
||||
*
|
||||
* Usage:
|
||||
* CLI: yarn run migratedb
|
||||
* Programmatic:
|
||||
* import { runMigrations } from './db/migrations/migrate.js';
|
||||
* await runMigrations();
|
||||
*
|
||||
* Migration file format (example: db/migrations/sql/1.add-users.js):
|
||||
* export function up(db) {
|
||||
* db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)");
|
||||
* }
|
||||
*
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import {pathToFileURL} from 'url';
|
||||
import crypto from 'crypto';
|
||||
import SqliteConnection from '../../lib/services/storage/SqliteConnection.js';
|
||||
import logger from '../../lib/services/logger.js';
|
||||
|
||||
const ROOT = path.resolve('.');
|
||||
const MIGRATIONS_DIR = path.join(ROOT, 'db', 'migrations', 'sql');
|
||||
|
||||
/**
|
||||
* Ensures that the given directory exists, creating it recursively if needed.
|
||||
* @param {string} p - Path to the directory.
|
||||
*/
|
||||
function ensureDir(p) {
|
||||
if (!fs.existsSync(p)) fs.mkdirSync(p, {recursive: true});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all migration files in the migrations directory.
|
||||
* Migration files must follow the format: <number>.<label>.js
|
||||
* @returns {Array<{id:number, name:string, label:string, path:string}>}
|
||||
*/
|
||||
function listMigrationFiles() {
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
return fs
|
||||
.readdirSync(MIGRATIONS_DIR)
|
||||
.filter((f) => /^\d+\..+\.js$/.test(f))
|
||||
.map((file) => {
|
||||
const [idStr, ...rest] = file.split('.');
|
||||
const id = Number.parseInt(idStr, 10);
|
||||
const label = rest.slice(0, -1).join('.');
|
||||
const fullPath = path.join(MIGRATIONS_DIR, file);
|
||||
return {id, name: file, label, path: fullPath};
|
||||
})
|
||||
.sort((a, b) => (a.id === b.id ? a.name.localeCompare(b.name) : a.id - b.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the SHA-256 checksum of a file.
|
||||
* @param {string} filePath - Path to the file.
|
||||
* @returns {string} Hex-encoded checksum.
|
||||
*/
|
||||
function sha256File(filePath) {
|
||||
const buf = fs.readFileSync(filePath);
|
||||
return crypto.createHash('sha256').update(buf).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamically imports a migration module and returns its `up` function.
|
||||
* @param {string} filePath - Path to the migration file.
|
||||
* @returns {Promise<Function>} Migration function.
|
||||
* @throws {Error} If the migration file does not export a valid function.
|
||||
*/
|
||||
async function loadMigrationModule(filePath) {
|
||||
const testImporter = globalThis.__TEST_MIGRATE_IMPORT__;
|
||||
const url = pathToFileURL(filePath);
|
||||
const mod = testImporter ? await testImporter(filePath, url) : await import(url.href);
|
||||
const fn = mod.up || mod.default;
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`Migration ${filePath} must export function up(db) or default function(db)`);
|
||||
}
|
||||
return fn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all previously executed migrations from the database.
|
||||
* @returns {Map<string,string>} Map of migration name to checksum.
|
||||
*/
|
||||
function loadExecutedMigrations() {
|
||||
const executed = new Map();
|
||||
const hasTable = SqliteConnection.tableExists('schema_migrations');
|
||||
if (!hasTable) return executed;
|
||||
const rows = SqliteConnection.query('SELECT name, checksum FROM schema_migrations ORDER BY applied_at ASC');
|
||||
for (const r of rows) executed.set(r.name, r.checksum);
|
||||
return executed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes all pending migrations.
|
||||
* Ensures that each migration runs inside its own transaction.
|
||||
* Already applied migrations are skipped, unless checksum updates are allowed.
|
||||
* On success, updates the schema_migrations table and runs PRAGMA optimize.
|
||||
*/
|
||||
export async function runMigrations() {
|
||||
ensureDir(path.join(ROOT, 'db'));
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
|
||||
const files = listMigrationFiles();
|
||||
if (files.length === 0) {
|
||||
logger.info('No migration files found under', MIGRATIONS_DIR);
|
||||
return;
|
||||
}
|
||||
|
||||
SqliteConnection.getConnection();
|
||||
|
||||
const executed = loadExecutedMigrations();
|
||||
|
||||
let appliedMigrations = 0;
|
||||
for (const m of files) {
|
||||
const checksum = sha256File(m.path);
|
||||
|
||||
if (executed.has(m.name)) {
|
||||
const prev = executed.get(m.name);
|
||||
if (prev !== checksum) {
|
||||
const allow = (process.env.MIGRATION_ALLOW_CHECKSUM_UPDATE || '').toLowerCase();
|
||||
const allowUpdate = allow === '1' || allow === 'true' || allow === 'yes';
|
||||
if (allowUpdate) {
|
||||
logger.warn(
|
||||
`Checksum mismatch for already executed migration ${m.name}, but MIGRATION_ALLOW_CHECKSUM_UPDATE is enabled. ` +
|
||||
`Updating recorded checksum and continuing without re-running the migration.`,
|
||||
);
|
||||
SqliteConnection.execute(
|
||||
'UPDATE schema_migrations SET checksum = @checksum WHERE name = @name',
|
||||
{checksum, name: m.name},
|
||||
);
|
||||
executed.set(m.name, checksum);
|
||||
} else {
|
||||
throw new Error(
|
||||
`Checksum mismatch for already executed migration ${m.name}. ` +
|
||||
`Do not modify applied migrations. Create a new migration instead.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
appliedMigrations++;
|
||||
logger.info(`Applying migration: ${m.name}`);
|
||||
const fn = await loadMigrationModule(m.path);
|
||||
|
||||
try {
|
||||
let duration = 0;
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const t0 = Date.now();
|
||||
fn(db);
|
||||
duration = Date.now() - t0;
|
||||
db
|
||||
.prepare(
|
||||
'INSERT INTO schema_migrations (name, checksum, applied_at, duration_ms) VALUES (?, ?, datetime(\'now\'), ?)',
|
||||
)
|
||||
.run(m.name, checksum, duration);
|
||||
});
|
||||
logger.info(`Migration applied: ${m.name} (${duration} ms)`);
|
||||
} catch (e) {
|
||||
logger.error(`Migration failed and was rolled back: ${m.name}`, e);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
SqliteConnection.optimize();
|
||||
if (appliedMigrations > 0) {
|
||||
logger.info('All migrations completed successfully.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects whether the current file is being executed directly via Node.js.
|
||||
* This allows `node db/migrations/migrate.js` to run migrations directly.
|
||||
* @returns {boolean} True if the file was run directly.
|
||||
*/
|
||||
const isDirectRun = (() => {
|
||||
try {
|
||||
const thisFile = import.meta.url;
|
||||
const invoked = pathToFileURL(process.argv[1] || '').href;
|
||||
return thisFile === invoked;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
if (isDirectRun) {
|
||||
await runMigrations();
|
||||
}
|
||||
|
Before Width: | Height: | Size: 121 KiB After Width: | Height: | Size: 197 KiB |
BIN
doc/screenshot2.png
Normal file
|
After Width: | Height: | Size: 512 KiB |
BIN
doc/screenshot3.png
Normal file
|
After Width: | Height: | Size: 372 KiB |
|
Before Width: | Height: | Size: 323 KiB |
|
Before Width: | Height: | Size: 93 KiB |
@@ -5,11 +5,18 @@ services:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: fredy/fredy
|
||||
image: ghcr.io/orangecoding/fredy
|
||||
# map existing config and database
|
||||
volumes:
|
||||
- ./conf:/conf
|
||||
- ./db:/db
|
||||
ports:
|
||||
- 9998:9998
|
||||
- "9998:9998"
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
# The container will immediately stop when health check fails after retries
|
||||
test: ["CMD-SHELL", "curl --fail --silent --show-error --max-time 5 http://localhost:9998/ || exit 1"]
|
||||
interval: 120s
|
||||
timeout: 10s
|
||||
retries: 1
|
||||
start_period: 10s
|
||||
|
||||
18
docker-test.sh
Normal file
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# Stop and remove old container if it exists
|
||||
if [ "$(docker ps -aq -f name=fredy)" ]; then
|
||||
docker stop fredy || true
|
||||
docker rm fredy || true
|
||||
fi
|
||||
|
||||
# Build image from local Dockerfile
|
||||
docker build -t fredy:local .
|
||||
|
||||
# Run container with volumes and port mapping
|
||||
docker run -d --name fredy \
|
||||
-v fredy_conf:/conf \
|
||||
-v fredy_db:/db \
|
||||
-p 9998:9998 \
|
||||
fredy:local
|
||||
92
index.js
@@ -1,15 +1,27 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { config } from './lib/utils.js';
|
||||
import { checkIfConfigIsAccessible, config, getProviders, refreshConfig } from './lib/utils.js';
|
||||
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
|
||||
import * as jobStorage from './lib/services/storage/jobStorage.js';
|
||||
import FredyRuntime from './lib/FredyRuntime.js';
|
||||
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
|
||||
import { runMigrations } from './db/migrations/migrate.js';
|
||||
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
|
||||
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
|
||||
import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
|
||||
import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
|
||||
import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
|
||||
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
|
||||
import logger from './lib/services/logger.js';
|
||||
import { bus } from './lib/services/events/event-bus.js';
|
||||
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
|
||||
|
||||
// Load configuration before any other startup steps
|
||||
await refreshConfig();
|
||||
|
||||
const isConfigAccessible = await checkIfConfigIsAccessible();
|
||||
|
||||
if (!isConfigAccessible) {
|
||||
logger.error('Configuration exists, but is not accessible. Please check the file permission');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
|
||||
const rawDir = config.sqlitepath || '/db';
|
||||
@@ -22,8 +34,9 @@ if (!fs.existsSync(absDir)) {
|
||||
// Run DB migrations once at startup and block until finished
|
||||
await runMigrations();
|
||||
|
||||
const providersPath = './lib/provider';
|
||||
const provider = fs.readdirSync(providersPath).filter((file) => file.endsWith('.js'));
|
||||
// Load provider modules once at startup
|
||||
const providers = await getProviders();
|
||||
|
||||
//assuming interval is always in minutes
|
||||
const INTERVAL = config.interval * 60 * 1000;
|
||||
|
||||
@@ -37,37 +50,46 @@ if (config.demoMode) {
|
||||
|
||||
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
|
||||
|
||||
const fetchedProvider = await Promise.all(
|
||||
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${providersPath}/${pro}`)),
|
||||
);
|
||||
|
||||
ensureAdminUserExists();
|
||||
ensureDemoUserExists();
|
||||
await initTrackerCron();
|
||||
//do not wait for this to finish, let it run in the background
|
||||
initActiveCheckerCron();
|
||||
|
||||
setInterval(
|
||||
(function exec() {
|
||||
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
|
||||
if (!config.demoMode) {
|
||||
if (isDuringWorkingHoursOrNotSet) {
|
||||
config.lastRun = Date.now();
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.forEach((job) => {
|
||||
job.provider
|
||||
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
|
||||
.forEach(async (prov) => {
|
||||
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
|
||||
pro.init(prov, job.blacklist);
|
||||
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||
}
|
||||
bus.on('jobs:runAll', () => {
|
||||
logger.debug('Running Fredy Job manually');
|
||||
execute();
|
||||
});
|
||||
|
||||
const execute = () => {
|
||||
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
|
||||
if (!config.demoMode) {
|
||||
if (isDuringWorkingHoursOrNotSet) {
|
||||
config.lastRun = Date.now();
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.forEach((job) => {
|
||||
job.provider
|
||||
.filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
|
||||
.forEach(async (prov) => {
|
||||
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
||||
matchedProvider.init(prov, job.blacklist);
|
||||
await new FredyRuntime(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
).execute();
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||
}
|
||||
return exec;
|
||||
})(),
|
||||
INTERVAL,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
setInterval(execute, INTERVAL);
|
||||
//start once at startup
|
||||
execute();
|
||||
|
||||
@@ -107,7 +107,7 @@ class FredyRuntime {
|
||||
}
|
||||
return !similar;
|
||||
});
|
||||
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, listings.address));
|
||||
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
|
||||
return filteredList;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,10 +3,11 @@ import { authInterceptor, cookieSession, adminInterceptor } from './security.js'
|
||||
import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
|
||||
import { analyticsRouter } from './routes/analyticsRouter.js';
|
||||
import { providerRouter } from './routes/providerRouter.js';
|
||||
import { versionRouter } from './routes/versionRouter.js';
|
||||
import { loginRouter } from './routes/loginRoute.js';
|
||||
import { config } from '../utils.js';
|
||||
import { userRouter } from './routes/userRoute.js';
|
||||
import { jobRouter } from './routes/jobRouter.js';
|
||||
import { config } from '../utils.js';
|
||||
import bodyParser from 'body-parser';
|
||||
import restana from 'restana';
|
||||
import files from 'serve-static';
|
||||
@@ -14,6 +15,7 @@ import path from 'path';
|
||||
import { getDirName } from '../utils.js';
|
||||
import { demoRouter } from './routes/demoRouter.js';
|
||||
import logger from '../services/logger.js';
|
||||
import { listingsRouter } from './routes/listingsRouter.js';
|
||||
const service = restana();
|
||||
const staticService = files(path.join(getDirName(), '../ui/public'));
|
||||
const PORT = config.port || 9998;
|
||||
@@ -23,6 +25,9 @@ service.use(cookieSession());
|
||||
service.use(staticService);
|
||||
service.use('/api/admin', authInterceptor());
|
||||
service.use('/api/jobs', authInterceptor());
|
||||
service.use('/api/version', authInterceptor());
|
||||
service.use('/api/listings', authInterceptor());
|
||||
|
||||
// /admin can only be accessed when user is having admin permissions
|
||||
service.use('/api/admin', adminInterceptor());
|
||||
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
|
||||
@@ -30,8 +35,10 @@ service.use('/api/admin/generalSettings', generalSettingsRouter);
|
||||
service.use('/api/jobs/provider', providerRouter);
|
||||
service.use('/api/jobs/insights', analyticsRouter);
|
||||
service.use('/api/admin/users', userRouter);
|
||||
service.use('/api/version', versionRouter);
|
||||
service.use('/api/jobs', jobRouter);
|
||||
service.use('/api/login', loginRouter);
|
||||
service.use('/api/listings', listingsRouter);
|
||||
//this route is unsecured intentionally as it is being queried from the login page
|
||||
service.use('/api/demo', demoRouter);
|
||||
|
||||
|
||||
@@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import { config } from '../../utils.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { bus } from '../../services/events/event-bus.js';
|
||||
|
||||
const service = restana();
|
||||
const jobRouter = service.newRouter();
|
||||
|
||||
function doesJobBelongsToUser(job, req) {
|
||||
const userId = req.session.currentUser;
|
||||
if (userId == null) {
|
||||
@@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) {
|
||||
}
|
||||
return user.isAdmin || job.userId === user.id;
|
||||
}
|
||||
|
||||
jobRouter.get('/', async (req, res) => {
|
||||
const isUserAdmin = isAdmin(req);
|
||||
//show only the jobs which belongs to the user (or all of the user is an admin)
|
||||
@@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => {
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.post('/startAll', async (req, res) => {
|
||||
bus.emit('jobs:runAll');
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.post('/', async (req, res) => {
|
||||
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
|
||||
try {
|
||||
|
||||
100
lib/api/routes/listingsRouter.js
Normal file
@@ -0,0 +1,100 @@
|
||||
import restana from 'restana';
|
||||
import * as listingStorage from '../../services/storage/listingsStorage.js';
|
||||
import * as watchListStorage from '../../services/storage/watchListStorage.js';
|
||||
import { isAdmin as isAdminFn } from '../security.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { nullOrEmpty } from '../../utils.js';
|
||||
import { getJobs } from '../../services/storage/jobStorage.js';
|
||||
|
||||
const service = restana();
|
||||
|
||||
const listingsRouter = service.newRouter();
|
||||
|
||||
listingsRouter.get('/table', async (req, res) => {
|
||||
const {
|
||||
page,
|
||||
pageSize = 50,
|
||||
activityFilter,
|
||||
jobNameFilter,
|
||||
providerFilter,
|
||||
watchListFilter,
|
||||
sortfield = null,
|
||||
sortdir = 'asc',
|
||||
freeTextFilter,
|
||||
} = req.query || {};
|
||||
|
||||
// normalize booleans (accept true, 'true', 1, '1')
|
||||
const toBool = (v) => v === true || v === 'true' || v === 1 || v === '1';
|
||||
const normalizedActivity = toBool(activityFilter) ? true : null;
|
||||
const normalizedWatch = toBool(watchListFilter) ? true : null;
|
||||
|
||||
let jobFilter = null;
|
||||
let jobIdFilter = null;
|
||||
const jobs = getJobs();
|
||||
if (!nullOrEmpty(jobNameFilter)) {
|
||||
const job = jobs.find((j) => j.id === jobNameFilter);
|
||||
jobFilter = job != null ? job.name : null;
|
||||
jobIdFilter = job != null ? job.id : null;
|
||||
}
|
||||
|
||||
res.body = listingStorage.queryListings({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
jobNameFilter: jobFilter,
|
||||
jobIdFilter: jobIdFilter,
|
||||
providerFilter,
|
||||
watchListFilter: normalizedWatch,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdminFn(req),
|
||||
});
|
||||
res.send();
|
||||
});
|
||||
|
||||
// Toggle watch state for the current user on a listing
|
||||
listingsRouter.post('/watch', async (req, res) => {
|
||||
try {
|
||||
const { listingId } = req.body || {};
|
||||
const userId = req.session?.currentUser;
|
||||
if (!listingId || !userId) {
|
||||
res.statusCode = 400;
|
||||
res.body = { message: 'listingId or user not provided' };
|
||||
return res.send();
|
||||
}
|
||||
watchListStorage.toggleWatch(listingId, userId);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
res.statusCode = 500;
|
||||
res.body = { message: 'Failed to toggle watch' };
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.delete('/job', async (req, res) => {
|
||||
const { jobId } = req.body;
|
||||
try {
|
||||
listingStorage.deleteListingsByJobId(jobId);
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.delete('/', async (req, res) => {
|
||||
const { ids } = req.body;
|
||||
try {
|
||||
if (Array.isArray(ids) && ids.length > 0) {
|
||||
listingStorage.deleteListingsById(ids);
|
||||
}
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
export { listingsRouter };
|
||||
38
lib/api/routes/versionRouter.js
Normal file
@@ -0,0 +1,38 @@
|
||||
import restana from 'restana';
|
||||
import fetch from 'node-fetch';
|
||||
import { getPackageVersion } from '../../utils.js';
|
||||
import semver from 'semver';
|
||||
|
||||
const service = restana();
|
||||
const versionRouter = service.newRouter();
|
||||
|
||||
versionRouter.get('/', async (req, res) => {
|
||||
const versionPayload = await getCurrentVersionFromGithub();
|
||||
const localFredyVersion = await getPackageVersion();
|
||||
res.body =
|
||||
versionPayload == null
|
||||
? {
|
||||
newVersion: false,
|
||||
localFredyVersion,
|
||||
}
|
||||
: versionPayload;
|
||||
res.send();
|
||||
});
|
||||
|
||||
async function getCurrentVersionFromGithub() {
|
||||
const raw = await fetch('https://api.github.com/repos/orangecoding/fredy/releases/latest');
|
||||
const data = await raw.json();
|
||||
const localFredyVersion = await getPackageVersion();
|
||||
if (data.tag_name == null || semver.gte(localFredyVersion, data.tag_name)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
newVersion: true,
|
||||
version: data.tag_name,
|
||||
url: data.html_url,
|
||||
body: data.body,
|
||||
localFredyVersion,
|
||||
};
|
||||
}
|
||||
|
||||
export { versionRouter };
|
||||
@@ -37,7 +37,7 @@ const cookieSession$0 = (userId) => {
|
||||
name: 'fredy-admin-session',
|
||||
keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
|
||||
userId,
|
||||
maxAge: 8 * 60 * 60 * 1000, // 8 hours
|
||||
maxAge: 2 * 60 * 60 * 1000, // 2 hours
|
||||
});
|
||||
};
|
||||
export { cookieSession$0 as cookieSession };
|
||||
|
||||
@@ -8,7 +8,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
const promises = newListings.map((newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
|
||||
return fetch(server, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
|
||||
130
lib/notification/adapter/discord_webhook.js
Normal file
@@ -0,0 +1,130 @@
|
||||
import fetch from 'node-fetch';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
/**
|
||||
* Generates an idempotent decimal color code. The input string-based color code is
|
||||
* generated using the djb2 hash algorithm.
|
||||
*
|
||||
* @param {string} str - Input string as color code base
|
||||
* @returns {number} Generated decimal color code (0 - 16777215)
|
||||
*/
|
||||
const generateColorFromString = (str) => {
|
||||
let hash = 5381; // initial value
|
||||
const input = String(str);
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
// hash * 33 + charCode
|
||||
hash = (hash << 5) + hash + input.charCodeAt(i);
|
||||
// Ensure the hash is 32 bit
|
||||
hash |= 0;
|
||||
}
|
||||
|
||||
let positiveHash = hash >>> 0;
|
||||
const maxColorValue = 16777215;
|
||||
const colorDecimal = positiveHash % maxColorValue;
|
||||
|
||||
return colorDecimal;
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates an embed per listing
|
||||
* (-> see https://birdie0.github.io/discord-webhooks-guide/structure/embeds.html).
|
||||
*
|
||||
* @param {string} jobKey - Key of job (used to set embed color)
|
||||
* @param {object} listing - Object holding listing details
|
||||
* @returns {object} Discord webhook embed
|
||||
*/
|
||||
const buildEmbed = (jobKey, listing) => {
|
||||
const maxTitleLength = 252; // Max embed title length is 256 characters
|
||||
let title = String(listing.title ?? 'N/A');
|
||||
if (title.length > maxTitleLength) {
|
||||
title = title.substring(0, maxTitleLength) + '...';
|
||||
}
|
||||
|
||||
const fields = [
|
||||
{
|
||||
name: 'Price',
|
||||
value: String(listing.price ?? 'n/a'),
|
||||
inline: true,
|
||||
},
|
||||
{
|
||||
name: 'Size',
|
||||
value: listing?.size?.replace(/2m/g, 'm²') ?? 'n/a',
|
||||
inline: true,
|
||||
},
|
||||
{
|
||||
name: 'Address',
|
||||
value: String(listing.address ?? 'n/a'),
|
||||
inline: true,
|
||||
},
|
||||
];
|
||||
|
||||
const embed = {
|
||||
title: title,
|
||||
color: generateColorFromString(jobKey),
|
||||
url: listing.link,
|
||||
fields: fields,
|
||||
};
|
||||
|
||||
if (listing.image) {
|
||||
embed.image = {
|
||||
url: normalizeImageUrl(listing.image),
|
||||
};
|
||||
}
|
||||
|
||||
return embed;
|
||||
};
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const adapter = notificationConfig.find((adapter) => adapter.id === config.id);
|
||||
const webhookUrl = adapter?.fields?.webhookUrl;
|
||||
if (!webhookUrl || newListings.length === 0) return Promise.resolve([]);
|
||||
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job?.name || jobKey;
|
||||
|
||||
const embeds = newListings.map((listing) => buildEmbed(jobKey, listing));
|
||||
|
||||
const maxEmbedsPerMessage = 10; // Discord only allows up to 10 embeds
|
||||
const webhookPromises = [];
|
||||
|
||||
for (let i = 0; i < embeds.length; i += maxEmbedsPerMessage) {
|
||||
// Send multiple Discord messages with up to 10 embeds per message
|
||||
const embedChunk = embeds.slice(i, i + maxEmbedsPerMessage);
|
||||
|
||||
const content = i === 0 ? `*${jobName}:* ${serviceName} found **${newListings.length}** new listings.` : '';
|
||||
const body = JSON.stringify({
|
||||
content: content,
|
||||
embeds: embedChunk,
|
||||
});
|
||||
|
||||
const fetchPromise = fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body,
|
||||
}).catch((error) => {
|
||||
console.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
|
||||
return Promise.reject(new Error(`Webhook failed: ${error.message}`));
|
||||
});
|
||||
|
||||
webhookPromises.push(fetchPromise);
|
||||
}
|
||||
|
||||
return Promise.allSettled(webhookPromises);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'discord_webhook',
|
||||
name: 'Discord Webhook',
|
||||
readme: markdown2Html('lib/notification/adapter/discord_webhook.md'),
|
||||
description: 'Fredy will send new listings to the Discord channel of your choice.',
|
||||
fields: {
|
||||
webhookUrl: {
|
||||
type: 'text',
|
||||
label: 'Webhook URL',
|
||||
description: 'The URL of the Discord webhook to send messages to.',
|
||||
},
|
||||
},
|
||||
};
|
||||
4
lib/notification/adapter/discord_webhook.md
Normal file
@@ -0,0 +1,4 @@
|
||||
### Discord Adapter
|
||||
|
||||
To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
|
||||
Once you have created a webhook, copy and paste the webhook URL.
|
||||
@@ -13,10 +13,10 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
||||
return fetch(webhook, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: {
|
||||
body: JSON.stringify({
|
||||
channel: channel,
|
||||
text: message,
|
||||
},
|
||||
}),
|
||||
});
|
||||
};
|
||||
export const config = {
|
||||
|
||||
@@ -15,11 +15,17 @@ Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$'
|
||||
Price: ${newListing.price}
|
||||
Link: ${newListing.link}`;
|
||||
|
||||
const sanitizeHeaderValue = (value) =>
|
||||
String(value ?? '')
|
||||
.replace(/[\r\n]+/g, ' ')
|
||||
.replace(/[^\x20-\x7E]/g, ' ')
|
||||
.trim();
|
||||
|
||||
const headers = {
|
||||
Title: newListing.title,
|
||||
Priority: String(priority),
|
||||
Tags: `${serviceName},${jobName}`,
|
||||
Click: newListing.link,
|
||||
Title: sanitizeHeaderValue(newListing.title),
|
||||
Priority: sanitizeHeaderValue(priority),
|
||||
Tags: sanitizeHeaderValue(`${serviceName},${jobName}`),
|
||||
Click: sanitizeHeaderValue(newListing.link),
|
||||
};
|
||||
|
||||
if (newListing.image && typeof newListing.image === 'string') {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
@@ -29,8 +30,8 @@ function normalizePrice(price) {
|
||||
return result[0];
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
@@ -49,6 +50,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -24,8 +25,8 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
@@ -46,6 +47,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
@@ -11,8 +12,8 @@ function normalize(o) {
|
||||
return Object.assign(o, { id, address, price, size, title, link });
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
const config = {
|
||||
@@ -28,9 +29,11 @@ const config = {
|
||||
address: 'div[data-testid="cardmfe-description-box-address"] | trim',
|
||||
image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
|
||||
link: 'button@data-base',
|
||||
description: 'div[data-testid="cardmfe-description-text-test-id"] | trim',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* The mobile API provides the following endpoints:
|
||||
* - GET /search/total?{search parameters}: Returns the total number of listings for the given query
|
||||
* Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
|
||||
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
|
||||
*
|
||||
* - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
|
||||
* data specifying additional results (advertisements) to return. The format is as follows:
|
||||
@@ -15,12 +15,12 @@
|
||||
* ```
|
||||
* It is not necessary to provide data for the specified keys.
|
||||
*
|
||||
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
|
||||
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
|
||||
|
||||
* - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
|
||||
* listing response.
|
||||
*
|
||||
* Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
|
||||
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
|
||||
*
|
||||
*
|
||||
* It is necessary to set the correct User Agent (see `getListings`) in the request header.
|
||||
@@ -35,8 +35,11 @@
|
||||
*
|
||||
*/
|
||||
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import {
|
||||
convertImmoscoutListingToMobileListing,
|
||||
convertWebToMobile,
|
||||
} from '../services/immoscout/immoscout-web-translator.js';
|
||||
import logger from '../services/logger.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -44,7 +47,7 @@ async function getListings(url) {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': 'ImmoScout24_1410_30_._',
|
||||
'User-Agent': 'ImmoScout_27.3_26.0_._',
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
@@ -77,6 +80,25 @@ async function getListings(url) {
|
||||
});
|
||||
}
|
||||
|
||||
async function isListingActive(link) {
|
||||
const result = await fetch(convertImmoscoutListingToMobileListing(link), {
|
||||
headers: {
|
||||
'User-Agent': 'ImmoScout_27.3_26.0_._',
|
||||
},
|
||||
});
|
||||
|
||||
if (result.status === 200) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (result.status === 404) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
logger.warn('Unknown status for immoscout listing', link);
|
||||
return -1;
|
||||
}
|
||||
|
||||
function nullOrEmpty(val) {
|
||||
return val == null || val.length === 0;
|
||||
}
|
||||
@@ -87,7 +109,7 @@ function normalize(o) {
|
||||
return Object.assign(o, { id, title, address });
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
return !utils.isOneOf(o.title, appliedBlackList);
|
||||
return !isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
const config = {
|
||||
url: null,
|
||||
@@ -104,6 +126,7 @@ const config = {
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
getListings: getListings,
|
||||
activeTester: isListingActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -14,8 +15,8 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
@@ -35,6 +36,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -8,8 +9,8 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
@@ -25,11 +26,13 @@ const config = {
|
||||
size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
|
||||
title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
|
||||
link: 'a@href',
|
||||
description: 'div[data-testid="cardmfe-description-text-test-id"] > div:nth-of-type(2) | removeNewline | trim',
|
||||
address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
|
||||
image: 'div[data-testid="cardMfe-card-pictureBox-opacity"] img@src',
|
||||
image: 'div[data-testid="cardmfe-picture-box-opacity-layer-test-id"] img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
let appliedBlacklistedDistricts = [];
|
||||
@@ -11,10 +12,10 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
const isBlacklistedDistrict =
|
||||
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
|
||||
appliedBlacklistedDistricts.length === 0 ? false : isOneOf(o.description, appliedBlacklistedDistricts);
|
||||
return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
@@ -36,6 +37,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const metaInformation = {
|
||||
name: 'Ebay Kleinanzeigen',
|
||||
|
||||
47
lib/provider/mcMakler.js
Executable file
@@ -0,0 +1,47 @@
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const originalId = o.id.split('/').pop();
|
||||
const id = buildHash(originalId, o.price);
|
||||
const size = o.size ?? 'N/A m²';
|
||||
const title = o.title || 'No title available';
|
||||
const address = o.address?.replace(' / ', ' ') || null;
|
||||
const link = o.link != null ? `https://www.mcmakler.de${o.link}` : config.url;
|
||||
return Object.assign(o, { id, size, title, link, address });
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: 'article[data-testid="propertyCard"]',
|
||||
sortByDateParam: 'sortBy=DATE&sortOn=DESC',
|
||||
waitForSelector: 'ul[data-testid="listsContainer"]',
|
||||
crawlFields: {
|
||||
id: 'h2 a@href',
|
||||
title: 'h2 a | removeNewline | trim',
|
||||
price: 'footer > p:first-of-type | trim',
|
||||
size: 'footer > p:nth-of-type(2) | trim',
|
||||
address: 'div > h2 + p | removeNewline | trim',
|
||||
image: 'img@src',
|
||||
link: 'h2 a@href',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
export const metaInformation = {
|
||||
name: 'McMakler',
|
||||
baseUrl: 'https://www.mcmakler.de/immobilien/',
|
||||
id: 'mcMakler',
|
||||
};
|
||||
export { config };
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -15,14 +16,14 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
return !utils.isOneOf(o.title, appliedBlackList);
|
||||
return !isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '.col-12.mb-4',
|
||||
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
|
||||
waitForSelector: '.nbk-section',
|
||||
waitForSelector: 'div[data-live-name-value="SearchList"]',
|
||||
crawlFields: {
|
||||
id: 'a@href',
|
||||
title: 'a@title | removeNewline | trim',
|
||||
@@ -33,6 +34,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
49
lib/provider/regionalimmobilien24.js
Executable file
@@ -0,0 +1,49 @@
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = buildHash(o.id, o.price);
|
||||
const address = o.address?.replace(/^adresse /i, '') ?? null;
|
||||
const title = o.title || 'No title available';
|
||||
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
|
||||
|
||||
var urlReg = new RegExp(/url\((.*?)\)/gim);
|
||||
const image = o.image != null ? urlReg.exec(o.image)[1] : null;
|
||||
return Object.assign(o, { id, address, title, link, image });
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '.listentry-content',
|
||||
sortByDateParam: null, // sort by date is standard
|
||||
waitForSelector: 'body',
|
||||
crawlFields: {
|
||||
id: '.listentry-iconbar-share@data-sid | trim',
|
||||
title: 'h2 | trim',
|
||||
price: '.listentry-details-price .listentry-details-v | trim',
|
||||
size: '.listentry-details-size .listentry-details-v | trim',
|
||||
address: '.listentry-adress | trim',
|
||||
image: '.listentry-img@style',
|
||||
link: '.shariff@data-url',
|
||||
description: '.listentry-extras | trim',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
export const metaInformation = {
|
||||
name: 'Regionalimmobilien24',
|
||||
baseUrl: 'https://www.regionalimmobilien24.de/',
|
||||
id: 'regionalimmobilien24',
|
||||
};
|
||||
export { config };
|
||||
46
lib/provider/sparkasse.js
Executable file
@@ -0,0 +1,46 @@
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const originalId = o.id.split('/').pop().replace('.html', '');
|
||||
const id = buildHash(originalId, o.price);
|
||||
const size = o.size?.replace(' Wohnfläche', '') ?? null;
|
||||
const title = o.title || 'No title available';
|
||||
const link = o.link != null ? `https://immobilien.sparkasse.de${o.link}` : config.url;
|
||||
return Object.assign(o, { id, size, title, link });
|
||||
}
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '.estate-list-item-row',
|
||||
sortByDateParam: 'sortBy=date_desc',
|
||||
waitForSelector: 'body',
|
||||
crawlFields: {
|
||||
id: 'div[data-testid="estate-link"] a@href',
|
||||
title: 'h3 | trim',
|
||||
price: '.estate-list-price | trim',
|
||||
size: '.estate-mainfact:first-child span | trim',
|
||||
address: 'h6 | trim',
|
||||
image: '.estate-list-item-image-container img@src',
|
||||
link: 'div[data-testid="estate-link"] a@href',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
export const metaInformation = {
|
||||
name: 'Sparkasse Immobilien',
|
||||
baseUrl: 'https://immobilien.sparkasse.de/',
|
||||
id: 'sparkasse',
|
||||
};
|
||||
export { config };
|
||||
@@ -1,4 +1,5 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -10,8 +11,8 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
@@ -31,6 +32,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
|
||||
23
lib/services/crons/demoCleanup-cron.js
Normal file
@@ -0,0 +1,23 @@
|
||||
import { removeJobsByUserId } from '../storage/jobStorage.js';
|
||||
import { config } from '../../utils.js';
|
||||
import { getUsers } from '../storage/userStorage.js';
|
||||
import logger from '../logger.js';
|
||||
import cron from 'node-cron';
|
||||
|
||||
/**
|
||||
* if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
|
||||
*/
|
||||
export function cleanupDemoAtMidnight() {
|
||||
cron.schedule('0 0 * * *', cleanup);
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
if (config.demoMode) {
|
||||
const demoUser = getUsers(false).find((user) => user.username === 'demo');
|
||||
if (demoUser == null) {
|
||||
logger.error('Demo user not found, cannot remove Jobs');
|
||||
return;
|
||||
}
|
||||
removeJobsByUserId(demoUser.id);
|
||||
}
|
||||
}
|
||||
13
lib/services/crons/listing-alive-cron.js
Normal file
@@ -0,0 +1,13 @@
|
||||
import cron from 'node-cron';
|
||||
import runActiveChecker from '../listings/listingActiveService.js';
|
||||
|
||||
async function runTask() {
|
||||
await runActiveChecker();
|
||||
}
|
||||
|
||||
export async function initActiveCheckerCron() {
|
||||
//run directly on start
|
||||
await runTask();
|
||||
// then every day at 1 am
|
||||
cron.schedule('0 1 * * *', runTask);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import cron from 'node-cron';
|
||||
import { config, inDevMode } from '../../utils.js';
|
||||
import { trackMainEvent } from './Tracker.js';
|
||||
import { trackMainEvent } from '../tracking/Tracker.js';
|
||||
|
||||
async function runTask() {
|
||||
//make sure to only send tracking events if the user gave us the green light and we are not in dev mode
|
||||
@@ -1,38 +0,0 @@
|
||||
import { setInterval } from 'node:timers';
|
||||
import { removeJobsByUserId } from './storage/jobStorage.js';
|
||||
import { config } from '../utils.js';
|
||||
import { getUsers } from './storage/userStorage.js';
|
||||
import logger from './logger.js';
|
||||
|
||||
/**
|
||||
* if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
|
||||
*/
|
||||
export function cleanupDemoAtMidnight() {
|
||||
const now = new Date();
|
||||
const millisUntilMidnightUTC =
|
||||
(24 - now.getUTCHours()) * 60 * 60 * 1000 -
|
||||
now.getUTCMinutes() * 60 * 1000 -
|
||||
now.getUTCSeconds() * 1000 -
|
||||
now.getUTCMilliseconds();
|
||||
|
||||
cleanup();
|
||||
setTimeout(() => {
|
||||
setInterval(
|
||||
() => {
|
||||
cleanup();
|
||||
},
|
||||
24 * 60 * 60 * 1000,
|
||||
);
|
||||
}, millisUntilMidnightUTC);
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
if (config.demoMode) {
|
||||
const demoUser = getUsers(false).find((user) => user.username === 'demo');
|
||||
if (demoUser == null) {
|
||||
logger.error('Demo user not found, cannot remove Jobs');
|
||||
return;
|
||||
}
|
||||
removeJobsByUserId(demoUser.id);
|
||||
}
|
||||
}
|
||||
2
lib/services/events/event-bus.js
Normal file
@@ -0,0 +1,2 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
export const bus = new EventEmitter();
|
||||
@@ -9,19 +9,19 @@ export function loadParser(text) {
|
||||
|
||||
export function parse(crawlContainer, crawlFields, text, url) {
|
||||
if (!text) {
|
||||
logger.warn('No content found for ', url);
|
||||
logger.debug('No content found for ', url);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!crawlContainer || !crawlFields) {
|
||||
logger.warn('Cannot parse, selector was empty for url ', url);
|
||||
logger.debug('Cannot parse, selector was empty for url ', url);
|
||||
return null;
|
||||
}
|
||||
|
||||
const result = [];
|
||||
|
||||
if ($(crawlContainer).length === 0) {
|
||||
logger.warn('No elements in crawl container found for url ', url);
|
||||
logger.debug('No elements in crawl container found for url ', url);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,30 +2,56 @@ import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
|
||||
import logger from '../logger.js';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
export default async function execute(url, waitForSelector, options) {
|
||||
let browser;
|
||||
let page;
|
||||
let result = null;
|
||||
let userDataDir;
|
||||
let removeUserDataDir = false;
|
||||
try {
|
||||
debug(`Sending request to ${url} using Puppeteer.`);
|
||||
|
||||
// Prepare a dedicated temporary userDataDir to avoid leaking /tmp/.org.chromium.* dirs
|
||||
if (options && options.userDataDir) {
|
||||
userDataDir = options.userDataDir;
|
||||
removeUserDataDir = !!options.cleanupUserDataDir;
|
||||
} else {
|
||||
const prefix = path.join(os.tmpdir(), 'puppeteer-fredy-');
|
||||
userDataDir = fs.mkdtempSync(prefix);
|
||||
removeUserDataDir = true;
|
||||
}
|
||||
|
||||
browser = await puppeteer.launch({
|
||||
headless: options.puppeteerHeadless ?? true,
|
||||
args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox'],
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-gpu',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-crash-reporter',
|
||||
],
|
||||
timeout: options.puppeteerTimeout || 30_000,
|
||||
userDataDir,
|
||||
});
|
||||
let page = await browser.newPage();
|
||||
page = await browser.newPage();
|
||||
await page.setExtraHTTPHeaders(DEFAULT_HEADER);
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
let pageSource;
|
||||
//if we're extracting data from a spa, we must wait for the selector
|
||||
// if we're extracting data from a SPA, we must wait for the selector
|
||||
if (waitForSelector != null) {
|
||||
await page.waitForSelector(waitForSelector);
|
||||
const selectorTimeout = options?.puppeteerSelectorTimeout ?? options?.puppeteerTimeout ?? 30_000;
|
||||
await page.waitForSelector(waitForSelector, { timeout: selectorTimeout });
|
||||
pageSource = await page.evaluate((selector) => {
|
||||
return document.querySelector(selector).innerHTML;
|
||||
const el = document.querySelector(selector);
|
||||
return el ? el.innerHTML : '';
|
||||
}, waitForSelector);
|
||||
} else {
|
||||
pageSource = await page.content();
|
||||
@@ -35,16 +61,35 @@ export default async function execute(url, waitForSelector, options) {
|
||||
|
||||
if (botDetected(pageSource, statusCode)) {
|
||||
logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
|
||||
return null;
|
||||
result = null;
|
||||
} else {
|
||||
result = pageSource || (await page.content());
|
||||
}
|
||||
|
||||
return await page.content();
|
||||
} catch (error) {
|
||||
logger.error('Error executing with puppeteer executor', error);
|
||||
return null;
|
||||
result = null;
|
||||
} finally {
|
||||
if (browser != null) {
|
||||
await browser.close();
|
||||
try {
|
||||
if (page) {
|
||||
await page.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
if (browser != null) {
|
||||
await browser.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
if (removeUserDataDir && userDataDir) {
|
||||
await fs.promises.rm(userDataDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T
|
||||
https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC?
|
||||
*/
|
||||
import queryString from 'query-string';
|
||||
import { nullOrEmpty } from '../../utils.js';
|
||||
|
||||
const PARAM_NAME_MAP = {
|
||||
heatingtypes: 'heatingtypes',
|
||||
@@ -193,3 +194,14 @@ export function convertWebToMobile(webUrl) {
|
||||
|
||||
return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
|
||||
}
|
||||
|
||||
export function convertImmoscoutListingToMobileListing(url) {
|
||||
if (nullOrEmpty(url)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return url.replace(
|
||||
/^https:\/\/www\.immobilienscout24\.de\/expose\//,
|
||||
'https://api.mobile.immobilienscout24.de/expose/',
|
||||
);
|
||||
}
|
||||
|
||||
104
lib/services/listings/listingActiveService.js
Normal file
@@ -0,0 +1,104 @@
|
||||
import { deactivateListings, getActiveOrUnknownListings } from '../storage/listingsStorage.js';
|
||||
import { getProviders } from '../../utils.js';
|
||||
import logger from '../../services/logger.js';
|
||||
|
||||
/**
|
||||
* Runs the active-listing checker:
|
||||
* 1) Loads all listings with unknown or active status.
|
||||
* 2) Resolves each listing's provider and calls its `activeTester(link)`.
|
||||
* 3) Collects listings that are no longer active and deactivates them in one batch.
|
||||
*
|
||||
* Concurrency: network-bound checks are executed with a configurable concurrency limit.
|
||||
*
|
||||
* @param {object} [opts]
|
||||
* @param {number} [opts.concurrency=8] Max number of parallel activeTester calls.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export default async function runActiveChecker(opts = {}) {
|
||||
const { concurrency = 4 } = opts;
|
||||
|
||||
const listings = getActiveOrUnknownListings();
|
||||
if (!Array.isArray(listings) || listings.length === 0) {
|
||||
logger.debug('No listings to check.');
|
||||
return;
|
||||
}
|
||||
|
||||
const providers = await getProviders();
|
||||
if (!Array.isArray(providers) || providers.length === 0) {
|
||||
logger.warn('No providers available. Skipping active checks.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Build a map for O(1) provider lookup by id
|
||||
/** @type {Record<string, any>} */
|
||||
const providerById = Object.create(null);
|
||||
for (const p of providers) {
|
||||
const id = p?.metaInformation?.id;
|
||||
if (id) providerById[id] = p;
|
||||
}
|
||||
|
||||
// Small generic mapLimit to cap concurrency without extra deps
|
||||
/**
|
||||
* @template T, R
|
||||
* @param {T[]} items
|
||||
* @param {number} limit
|
||||
* @param {(item: T, index: number) => Promise<R>} worker
|
||||
* @returns {Promise<R[]>}
|
||||
*/
|
||||
async function mapLimit(items, limit, worker) {
|
||||
const results = new Array(items.length);
|
||||
let next = 0;
|
||||
|
||||
async function runOne() {
|
||||
while (next < items.length) {
|
||||
const i = next++;
|
||||
try {
|
||||
results[i] = await worker(items[i], i);
|
||||
} catch (err) {
|
||||
results[i] = /** @type {any} */ (err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const runners = Array.from({ length: Math.min(limit, items.length) }, runOne);
|
||||
await Promise.all(runners);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** @type {string[]} */
|
||||
const listingsSetToInactive = [];
|
||||
|
||||
await mapLimit(listings, concurrency, async (listing) => {
|
||||
const { provider: listingProviderId, link, id } = listing || {};
|
||||
|
||||
const matchedProvider = providerById[listingProviderId];
|
||||
if (!matchedProvider) {
|
||||
logger.warn('Could not find matching provider for', listingProviderId);
|
||||
return;
|
||||
}
|
||||
const tester = matchedProvider?.config?.activeTester;
|
||||
if (typeof tester !== 'function') {
|
||||
logger.warn('No activeTester configured for', listingProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Contract: activeTester(link) returns 1 if active, 0 if inactive
|
||||
let result;
|
||||
try {
|
||||
result = await tester(link);
|
||||
} catch {
|
||||
result = -1;
|
||||
}
|
||||
|
||||
if (result === 0 && id) {
|
||||
listingsSetToInactive.push(id);
|
||||
}
|
||||
});
|
||||
|
||||
if (listingsSetToInactive.length > 0) {
|
||||
logger.info(`Setting ${listingsSetToInactive.length} listings to inactive.`);
|
||||
deactivateListings(listingsSetToInactive);
|
||||
} else {
|
||||
logger.debug('No listings need to be set inactive.');
|
||||
}
|
||||
}
|
||||
68
lib/services/listings/listingActiveTester.js
Normal file
@@ -0,0 +1,68 @@
|
||||
import fetch from 'node-fetch';
|
||||
import { randomBetween, sleep } from '../../utils.js';
|
||||
|
||||
const maxAttempts = 3;
|
||||
|
||||
/**
|
||||
* Check if a listing is still active with up to 3 attempts and exponential backoff.
|
||||
* Backoff waits are capped and the last wait is at most 2000 ms.
|
||||
*
|
||||
* Rules:
|
||||
* - HTTP 200 => return 1
|
||||
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
|
||||
* - HTTP 404 => return 0
|
||||
* - Other statuses or network errors => retry until attempts are exhausted
|
||||
*
|
||||
* @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
|
||||
*/
|
||||
export default async function checkIfListingIsActive(link) {
|
||||
await sleep(randomBetween(50, 100));
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
const res = await fetch(link, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
|
||||
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.status === 200) {
|
||||
return 1;
|
||||
}
|
||||
if (res.status === 401) return -1;
|
||||
if (res.status === 403) return -1;
|
||||
if (res.status === 404) return 0;
|
||||
|
||||
// For any other status, only retry if attempts remain
|
||||
if (attempt < maxAttempts) {
|
||||
await sleep(backoffDelay(attempt));
|
||||
continue;
|
||||
}
|
||||
|
||||
return 0;
|
||||
} catch {
|
||||
// Network error: retry if attempts remain
|
||||
if (attempt < maxAttempts) {
|
||||
await sleep(backoffDelay(attempt));
|
||||
continue;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exponential backoff delay with cap.
|
||||
* attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
|
||||
* @param {number} attempt 1-based attempt index
|
||||
* @returns {number} delay in ms
|
||||
*/
|
||||
function backoffDelay(attempt) {
|
||||
const base = 500;
|
||||
const cap = 2000;
|
||||
return Math.min(base * 2 ** (attempt - 1), cap);
|
||||
}
|
||||
@@ -48,11 +48,44 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => {
|
||||
return SqliteConnection.query(
|
||||
`SELECT hash
|
||||
FROM listings
|
||||
WHERE job_id = @jobId AND provider = @providerId`,
|
||||
WHERE job_id = @jobId
|
||||
AND provider = @providerId`,
|
||||
{ jobId, providerId },
|
||||
).map((r) => r.hash);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return a list of listing that either are active or have an unknown status
|
||||
* to constantly check if they are still online
|
||||
*
|
||||
* @returns {string[]} Array of listings
|
||||
*/
|
||||
export const getActiveOrUnknownListings = () => {
|
||||
return SqliteConnection.query(
|
||||
`SELECT *
|
||||
FROM listings
|
||||
WHERE is_active is null
|
||||
OR is_active = 1
|
||||
ORDER BY provider`,
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Deactivates listings by setting is_active = 0 for all matching IDs.
|
||||
*
|
||||
* @param {string[]} ids - Array of listing IDs to deactivate.
|
||||
* @returns {object[]} Result of the SQLite query execution.
|
||||
*/
|
||||
export const deactivateListings = (ids) => {
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
return SqliteConnection.execute(
|
||||
`UPDATE listings
|
||||
SET is_active = 0
|
||||
WHERE id IN (${placeholders})`,
|
||||
ids,
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Persist a batch of scraped listings for a given job and provider.
|
||||
*
|
||||
@@ -85,11 +118,11 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
|
||||
link, created_at)
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
|
||||
@created_at)
|
||||
ON CONFLICT(hash) DO NOTHING`,
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
|
||||
link, created_at, is_active)
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
|
||||
@created_at, 1)
|
||||
ON CONFLICT(job_id, hash) DO NOTHING`,
|
||||
);
|
||||
|
||||
for (const item of listings) {
|
||||
@@ -136,3 +169,163 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
return str.replace(/\s*\([^)]*\)/g, '');
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Query listings with pagination, filtering and sorting.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {number} [params.pageSize=50]
|
||||
* @param {number} [params.page=1]
|
||||
* @param {string} [params.freeTextFilter]
|
||||
* @param {object} [params.activityFilter]
|
||||
* @param {object} [params.jobNameFilter]
|
||||
* @param {object} [params.providerFilter]
|
||||
* @param {object} [params.watchListFilter]
|
||||
* @param {string|null} [params.sortField=null] - One of: 'created_at','price','size','provider','title'.
|
||||
* @param {('asc'|'desc')} [params.sortDir='asc']
|
||||
* @param {string} [params.userId] - Current user id used to scope listings (ignored for admins).
|
||||
* @param {boolean} [params.isAdmin=false] - When true, returns all listings.
|
||||
* @returns {{ totalNumber:number, page:number, result:Object[] }}
|
||||
*/
|
||||
export const queryListings = ({
|
||||
pageSize = 50,
|
||||
page = 1,
|
||||
activityFilter,
|
||||
jobNameFilter,
|
||||
jobIdFilter,
|
||||
providerFilter,
|
||||
watchListFilter,
|
||||
freeTextFilter,
|
||||
sortField = null,
|
||||
sortDir = 'asc',
|
||||
userId = null,
|
||||
isAdmin = false,
|
||||
} = {}) => {
|
||||
// sanitize inputs
|
||||
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
|
||||
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
|
||||
const offset = (safePage - 1) * safePageSize;
|
||||
|
||||
// build WHERE filter across common text columns
|
||||
const whereParts = [];
|
||||
const params = { limit: safePageSize, offset };
|
||||
// always provide userId param for watched-flag evaluation (null -> no matches)
|
||||
params.userId = userId || '__NO_USER__';
|
||||
// user scoping (non-admin only): restrict to listings whose job belongs to user
|
||||
if (!isAdmin) {
|
||||
whereParts.push(`(j.user_id = @userId)`);
|
||||
}
|
||||
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
|
||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
|
||||
}
|
||||
// activityFilter: when true -> only active listings (is_active = 1)
|
||||
if (activityFilter === true) {
|
||||
whereParts.push('(is_active = 1)');
|
||||
}
|
||||
// Prefer filtering by job id when provided (unambiguous and robust)
|
||||
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
|
||||
params.jobId = String(jobIdFilter).trim();
|
||||
whereParts.push('(l.job_id = @jobId)');
|
||||
} else if (jobNameFilter && String(jobNameFilter).trim().length > 0) {
|
||||
// Fallback to exact job name match
|
||||
params.jobName = String(jobNameFilter).trim();
|
||||
whereParts.push('(j.name = @jobName)');
|
||||
}
|
||||
// providerFilter: when provided as string (assumed provider name), filter listings where provider equals that name (exact match)
|
||||
if (providerFilter && String(providerFilter).trim().length > 0) {
|
||||
params.providerName = String(providerFilter).trim();
|
||||
whereParts.push('(provider = @providerName)');
|
||||
}
|
||||
// watchListFilter: when true -> only watched listings
|
||||
if (watchListFilter === true) {
|
||||
whereParts.push('(wl.id IS NOT NULL)');
|
||||
}
|
||||
|
||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
const whereSqlWithAlias = whereSql
|
||||
.replace(/\btitle\b/g, 'l.title')
|
||||
.replace(/\bdescription\b/g, 'l.description')
|
||||
.replace(/\baddress\b/g, 'l.address')
|
||||
.replace(/\bprovider\b/g, 'l.provider')
|
||||
.replace(/\blink\b/g, 'l.link')
|
||||
.replace(/\bis_active\b/g, 'l.is_active')
|
||||
.replace(/\bj\.user_id\b/g, 'j.user_id')
|
||||
.replace(/\bj\.name\b/g, 'j.name')
|
||||
.replace(/\bwl\.id\b/g, 'wl.id');
|
||||
|
||||
// whitelist sortable fields to avoid SQL injection
|
||||
const sortable = new Set(['created_at', 'price', 'size', 'provider', 'title', 'job_name', 'is_active', 'isWatched']);
|
||||
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
|
||||
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
|
||||
const orderSql = safeSortField ? `ORDER BY ${safeSortField} ${safeSortDir}` : 'ORDER BY created_at DESC';
|
||||
const orderSqlWithAlias = orderSql
|
||||
.replace(/\bcreated_at\b/g, 'l.created_at')
|
||||
.replace(/\bprice\b/g, 'l.price')
|
||||
.replace(/\bsize\b/g, 'l.size')
|
||||
.replace(/\bprovider\b/g, 'l.provider')
|
||||
.replace(/\btitle\b/g, 'l.title')
|
||||
.replace(/\bjob_name\b/g, 'j.name')
|
||||
// Sort by computed watch flag when requested
|
||||
.replace(/\bisWatched\b/g, 'CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END');
|
||||
|
||||
// count total with same WHERE
|
||||
const countRow = SqliteConnection.query(
|
||||
`SELECT COUNT(1) as cnt
|
||||
FROM listings l
|
||||
LEFT JOIN jobs j ON j.id = l.job_id
|
||||
LEFT JOIN watch_list wl ON wl.listing_id = l.id AND wl.user_id = @userId
|
||||
${whereSqlWithAlias}`,
|
||||
params,
|
||||
);
|
||||
const totalNumber = countRow?.[0]?.cnt ?? 0;
|
||||
|
||||
// fetch page
|
||||
const rows = SqliteConnection.query(
|
||||
`SELECT l.*,
|
||||
j.name AS job_name,
|
||||
CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END AS isWatched
|
||||
FROM listings l
|
||||
LEFT JOIN jobs j ON j.id = l.job_id
|
||||
LEFT JOIN watch_list wl ON wl.listing_id = l.id AND wl.user_id = @userId
|
||||
${whereSqlWithAlias}
|
||||
${orderSqlWithAlias}
|
||||
LIMIT @limit OFFSET @offset`,
|
||||
params,
|
||||
);
|
||||
|
||||
return { totalNumber, page: safePage, result: rows };
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete all listings for a given job id.
|
||||
*
|
||||
* @param {string} jobId - The job identifier whose listings should be removed.
|
||||
* @returns {any} The result from SqliteConnection.execute (may contain changes count).
|
||||
*/
|
||||
export const deleteListingsByJobId = (jobId) => {
|
||||
if (!jobId) return;
|
||||
return SqliteConnection.execute(
|
||||
`DELETE
|
||||
FROM listings
|
||||
WHERE job_id = @jobId`,
|
||||
{ jobId },
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete listings by a list of listing IDs.
|
||||
*
|
||||
* @param {string[]} ids - Array of listing IDs to delete.
|
||||
* @returns {any} The result from SqliteConnection.execute.
|
||||
*/
|
||||
export const deleteListingsById = (ids) => {
|
||||
if (!Array.isArray(ids) || ids.length === 0) return;
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
return SqliteConnection.execute(
|
||||
`DELETE
|
||||
FROM listings
|
||||
WHERE id IN (${placeholders})`,
|
||||
ids,
|
||||
);
|
||||
};
|
||||
|
||||
185
lib/services/storage/migrations/migrate.js
Normal file
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* Migration Runner for better-sqlite3
|
||||
* I know there are external libs out there, but
|
||||
* a) most of them are pretty bloated
|
||||
* b) I wanted to have something that fit's this limited use-case
|
||||
* c) I was searching for justifications anyway to build a migration system on my own. Don't judge me ;)
|
||||
*
|
||||
* Executes all migration files in lib/services/storage/migrations/sql in natural order.
|
||||
* Each migration runs in its own transaction. If a migration fails, only that
|
||||
* migration is rolled back and the process stops with a non-zero exit code.
|
||||
* Already applied migrations are skipped using the schema_migrations table.
|
||||
*
|
||||
* Usage:
|
||||
* CLI: yarn run migratedb
|
||||
* Programmatic:
|
||||
* import { runMigrations } from './lib/services/storage/migrations/migrate.js';
|
||||
* await runMigrations();
|
||||
*
|
||||
* Migration file format (example: lib/services/storage/migrations/sql/1.add-users.js):
|
||||
* export function up(db) {
|
||||
* db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)");
|
||||
* }
|
||||
*
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import crypto from 'crypto';
|
||||
import SqliteConnection from '../SqliteConnection.js';
|
||||
import logger from '../../logger.js';
|
||||
|
||||
const ROOT = path.resolve('.');
|
||||
const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
|
||||
|
||||
/**
|
||||
* Ensures that the given directory exists, creating it recursively if needed.
|
||||
* @param {string} p - Path to the directory.
|
||||
*/
|
||||
function ensureDir(p) {
|
||||
if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all migration files in the migrations directory.
|
||||
* Migration files must follow the format: <number>.<label>.js
|
||||
* @returns {Array<{id:number, name:string, label:string, path:string}>}
|
||||
*/
|
||||
function listMigrationFiles() {
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
return fs
|
||||
.readdirSync(MIGRATIONS_DIR)
|
||||
.filter((f) => /^\d+\..+\.js$/.test(f))
|
||||
.map((file) => {
|
||||
const [idStr, ...rest] = file.split('.');
|
||||
const id = Number.parseInt(idStr, 10);
|
||||
const label = rest.slice(0, -1).join('.');
|
||||
const fullPath = path.join(MIGRATIONS_DIR, file);
|
||||
return { id, name: file, label, path: fullPath };
|
||||
})
|
||||
.sort((a, b) => (a.id === b.id ? a.name.localeCompare(b.name) : a.id - b.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the SHA-256 checksum of a file.
|
||||
* @param {string} filePath - Path to the file.
|
||||
* @returns {string} Hex-encoded checksum.
|
||||
*/
|
||||
function sha256File(filePath) {
|
||||
const buf = fs.readFileSync(filePath);
|
||||
return crypto.createHash('sha256').update(buf).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamically imports a migration module and returns its `up` function.
|
||||
* @param {string} filePath - Path to the migration file.
|
||||
* @returns {Promise<Function>} Migration function.
|
||||
* @throws {Error} If the migration file does not export a valid function.
|
||||
*/
|
||||
async function loadMigrationModule(filePath) {
|
||||
const testImporter = globalThis.__TEST_MIGRATE_IMPORT__;
|
||||
const url = pathToFileURL(filePath);
|
||||
const mod = testImporter ? await testImporter(filePath, url) : await import(url.href);
|
||||
const fn = mod.up || mod.default;
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`Migration ${filePath} must export function up(db) or default function(db)`);
|
||||
}
|
||||
return fn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all previously executed migrations from the database.
|
||||
* @returns {Map<string,string>} Map of migration name to checksum.
|
||||
*/
|
||||
function loadExecutedMigrations() {
|
||||
const executed = new Map();
|
||||
const hasTable = SqliteConnection.tableExists('schema_migrations');
|
||||
if (!hasTable) return executed;
|
||||
const rows = SqliteConnection.query('SELECT name, checksum FROM schema_migrations ORDER BY applied_at ASC');
|
||||
for (const r of rows) executed.set(r.name, r.checksum);
|
||||
return executed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes all pending migrations.
|
||||
* Ensures that each migration runs inside its own transaction.
|
||||
* Already applied migrations are skipped, unless checksum updates are allowed.
|
||||
* On success, updates the schema_migrations table and runs PRAGMA optimize.
|
||||
*/
|
||||
export async function runMigrations() {
|
||||
ensureDir(path.join(ROOT, 'db'));
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
|
||||
const files = listMigrationFiles();
|
||||
if (files.length === 0) {
|
||||
logger.info('No migration files found under', MIGRATIONS_DIR);
|
||||
return;
|
||||
}
|
||||
|
||||
SqliteConnection.getConnection();
|
||||
|
||||
const executed = loadExecutedMigrations();
|
||||
|
||||
let appliedMigrations = 0;
|
||||
for (const m of files) {
|
||||
const checksum = sha256File(m.path);
|
||||
|
||||
if (executed.has(m.name)) {
|
||||
const prev = executed.get(m.name);
|
||||
if (prev !== checksum) {
|
||||
logger.info(`Mismatch found in migration ${m.name}. Fixing.`);
|
||||
SqliteConnection.execute('UPDATE schema_migrations SET checksum = @checksum WHERE name = @name', {
|
||||
checksum,
|
||||
name: m.name,
|
||||
});
|
||||
executed.set(m.name, checksum);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
appliedMigrations++;
|
||||
logger.info(`Applying migration: ${m.name}`);
|
||||
const fn = await loadMigrationModule(m.path);
|
||||
|
||||
try {
|
||||
let duration = 0;
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const t0 = Date.now();
|
||||
fn(db);
|
||||
duration = Date.now() - t0;
|
||||
db.prepare(
|
||||
"INSERT INTO schema_migrations (name, checksum, applied_at, duration_ms) VALUES (?, ?, datetime('now'), ?)",
|
||||
).run(m.name, checksum, duration);
|
||||
});
|
||||
logger.info(`Migration applied: ${m.name} (${duration} ms)`);
|
||||
} catch (e) {
|
||||
logger.error(`Migration failed and was rolled back: ${m.name}`, e);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
SqliteConnection.optimize();
|
||||
if (appliedMigrations > 0) {
|
||||
logger.info('All migrations completed successfully.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects whether the current file is being executed directly via Node.js.
|
||||
* This allows `node lib/services/storage/migrations/migrate.js` to run migrations directly.
|
||||
* @returns {boolean} True if the file was run directly.
|
||||
*/
|
||||
const isDirectRun = (() => {
|
||||
try {
|
||||
const thisFile = import.meta.url;
|
||||
const invoked = pathToFileURL(process.argv[1] || '').href;
|
||||
return thisFile === invoked;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
if (isDirectRun) {
|
||||
await runMigrations();
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { toJson } from '../../../lib/utils.js';
|
||||
import { toJson } from '../../../../utils.js';
|
||||
|
||||
export function up(db) {
|
||||
// 1) Create tables
|
||||
@@ -69,7 +69,7 @@ export function up(db) {
|
||||
if (arr.length > 0) {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO users (id, username, password, last_login, is_admin)
|
||||
VALUES (@id, @username, @password, @last_login, @is_admin)`
|
||||
VALUES (@id, @username, @password, @last_login, @is_admin)`,
|
||||
);
|
||||
for (const u of arr) {
|
||||
stmt.run({
|
||||
@@ -77,7 +77,7 @@ export function up(db) {
|
||||
username: u.username,
|
||||
password: u.password,
|
||||
last_login: u.lastLogin ?? null,
|
||||
is_admin: u.isAdmin ? 1 : 0
|
||||
is_admin: u.isAdmin ? 1 : 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -96,7 +96,7 @@ export function up(db) {
|
||||
if (arr.length > 0) {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter)
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter)`
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter)`,
|
||||
);
|
||||
for (const j of arr) {
|
||||
stmt.run({
|
||||
@@ -106,7 +106,7 @@ export function up(db) {
|
||||
name: j.name ?? null,
|
||||
blacklist: toJson(j.blacklist ?? []),
|
||||
provider: toJson(j.provider ?? []),
|
||||
notification_adapter: toJson(j.notificationAdapter ?? [])
|
||||
notification_adapter: toJson(j.notificationAdapter ?? []),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
// Migration: there needs to be a unique index on job_id and hash as only
|
||||
// this makes the listing indeed unique
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN is_active INTEGER DEFAULT 1;
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
// Migration: there needs to be a unique index on job_id and hash as only
|
||||
// this makes the listing indeed unique
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
DROP INDEX IF EXISTS idx_listings_hash;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
|
||||
ON listings (job_id, hash);
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
// Migration: Adding a changeset field to the listings table in preparation for
|
||||
// a price watch feature
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN change_set jsonb;
|
||||
`);
|
||||
}
|
||||
15
lib/services/storage/migrations/sql/4.watch-list.js
Normal file
@@ -0,0 +1,15 @@
|
||||
// Migration: Adding a new table to store if somebody "watches" (a.k.a favorite) a listing
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS watch_list
|
||||
(
|
||||
id TEXT PRIMARY KEY,
|
||||
listing_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
FOREIGN KEY (listing_id) REFERENCES listings (id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_watch_list ON watch_list (listing_id, user_id);
|
||||
`);
|
||||
}
|
||||
64
lib/services/storage/watchListStorage.js
Normal file
@@ -0,0 +1,64 @@
|
||||
import SqliteConnection from './SqliteConnection.js';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
/**
|
||||
* Create a watch entry. Idempotent due to unique index (listing_id, user_id).
|
||||
* @param {string} listingId
|
||||
* @param {string} userId
|
||||
* @returns {{created:boolean}}
|
||||
*/
|
||||
export const createWatch = (listingId, userId) => {
|
||||
if (!listingId || !userId) return { created: false };
|
||||
try {
|
||||
SqliteConnection.execute(
|
||||
`INSERT INTO watch_list (id, listing_id, user_id)
|
||||
VALUES (@id, @listing_id, @user_id)
|
||||
ON CONFLICT(listing_id, user_id) DO NOTHING`,
|
||||
{ id: nanoid(), listing_id: listingId, user_id: userId },
|
||||
);
|
||||
// check whether it exists now
|
||||
const row = SqliteConnection.query(
|
||||
`SELECT 1 AS ok FROM watch_list WHERE listing_id = @listing_id AND user_id = @user_id LIMIT 1`,
|
||||
{ listing_id: listingId, user_id: userId },
|
||||
);
|
||||
return { created: row.length > 0 };
|
||||
} catch {
|
||||
return { created: false };
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete a watch entry.
|
||||
* @param {string} listingId
|
||||
* @param {string} userId
|
||||
* @returns {{deleted:boolean}}
|
||||
*/
|
||||
export const deleteWatch = (listingId, userId) => {
|
||||
if (!listingId || !userId) return { deleted: false };
|
||||
const res = SqliteConnection.execute(`DELETE FROM watch_list WHERE listing_id = @listing_id AND user_id = @user_id`, {
|
||||
listing_id: listingId,
|
||||
user_id: userId,
|
||||
});
|
||||
return { deleted: Boolean(res?.changes) };
|
||||
};
|
||||
|
||||
/**
|
||||
* Toggle a watch entry. If exists -> delete, otherwise create.
|
||||
* @param {string} listingId
|
||||
* @param {string} userId
|
||||
* @returns {{watched:boolean}}
|
||||
*/
|
||||
export const toggleWatch = (listingId, userId) => {
|
||||
if (!listingId || !userId) return { watched: false };
|
||||
const exists =
|
||||
SqliteConnection.query(
|
||||
`SELECT 1 AS ok FROM watch_list WHERE listing_id = @listing_id AND user_id = @user_id LIMIT 1`,
|
||||
{ listing_id: listingId, user_id: userId },
|
||||
).length > 0;
|
||||
if (exists) {
|
||||
deleteWatch(listingId, userId);
|
||||
return { watched: false };
|
||||
}
|
||||
createWatch(listingId, userId);
|
||||
return { watched: true };
|
||||
};
|
||||
@@ -1,9 +1,7 @@
|
||||
import { getJobs } from '../storage/jobStorage.js';
|
||||
import { getUniqueId } from './uniqueId.js';
|
||||
import { config, inDevMode } from '../../utils.js';
|
||||
import { config, getPackageVersion, inDevMode } from '../../utils.js';
|
||||
import os from 'os';
|
||||
import { readFileSync } from 'fs';
|
||||
import { packageUp } from 'package-up';
|
||||
import fetch from 'node-fetch';
|
||||
import logger from '../logger.js';
|
||||
|
||||
@@ -77,15 +75,3 @@ function enrichTrackingObject(trackingObject) {
|
||||
version,
|
||||
};
|
||||
}
|
||||
|
||||
async function getPackageVersion() {
|
||||
try {
|
||||
const packagePath = await packageUp();
|
||||
const packageJson = readFileSync(packagePath, 'utf8');
|
||||
const json = JSON.parse(packageJson);
|
||||
return json.version;
|
||||
} catch (error) {
|
||||
logger.error('Error reading version from package.json', error);
|
||||
}
|
||||
return 'N/A';
|
||||
}
|
||||
|
||||
135
lib/utils.js
@@ -1,15 +1,37 @@
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { createHash } from 'crypto';
|
||||
import { DEFAULT_CONFIG } from './defaultConfig.js';
|
||||
import fs from 'fs';
|
||||
import fs, { readFileSync } from 'fs';
|
||||
import logger from './services/logger.js';
|
||||
import { packageUp } from 'package-up';
|
||||
|
||||
const RE_GT = />/g;
|
||||
const RE_WEBP = /\/format\/webp/gi;
|
||||
const RE_EXT = /\.(jpe?g|png|gif)(\?.*)?$/i;
|
||||
const HTTPS_PREFIX = 'https://';
|
||||
const providersDirectoryPath = `${getDirName()}/provider`;
|
||||
|
||||
/**
|
||||
* Lazily load all provider modules from the provider directory.
|
||||
* Caches the resolved array to avoid re-importing on subsequent calls.
|
||||
*
|
||||
* @returns {Promise<any[]>} A list of loaded provider modules.
|
||||
*/
|
||||
let cachedProvidersPromise = null;
|
||||
|
||||
export function getProviders() {
|
||||
if (!cachedProvidersPromise) {
|
||||
/** @type {string[]} */
|
||||
const providerFileNames = fs.readdirSync(providersDirectoryPath).filter((fileName) => fileName.endsWith('.js'));
|
||||
cachedProvidersPromise = Promise.all(
|
||||
providerFileNames.map((fileName) => import(pathToFileURL(path.join(providersDirectoryPath, fileName)).href)),
|
||||
);
|
||||
}
|
||||
return cachedProvidersPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely stringify a value to JSON for storage.
|
||||
@@ -20,7 +42,7 @@ const HTTPS_PREFIX = 'https://';
|
||||
* @param {T} v - Any JSON-serializable value.
|
||||
* @returns {string|null} JSON string or null.
|
||||
*/
|
||||
export const toJson = (v) => (v == null ? null : JSON.stringify(v));
|
||||
const toJson = (v) => (v == null ? null : JSON.stringify(v));
|
||||
|
||||
/**
|
||||
* Safely parse JSON text coming from storage.
|
||||
@@ -32,7 +54,7 @@ export const toJson = (v) => (v == null ? null : JSON.stringify(v));
|
||||
* @param {T} fallback - Value to return when txt is null/invalid.
|
||||
* @returns {T} Parsed value or fallback.
|
||||
*/
|
||||
export const fromJson = (txt, fallback) => {
|
||||
const fromJson = (txt, fallback) => {
|
||||
if (txt == null) return fallback;
|
||||
try {
|
||||
return JSON.parse(txt);
|
||||
@@ -87,11 +109,22 @@ function timeStringToMs(timeString, now) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether current time is within configured working hours, or no hours are set.
|
||||
* If working hours are missing or incomplete, returns true.
|
||||
* @param {{workingHours?: {from?: string, to?: string}}} config
|
||||
* @param {number} now - Epoch ms
|
||||
* @returns {boolean}
|
||||
* Determine whether the given timestamp is within the configured working hours, or return true when the window is not set.
|
||||
* - If workingHours is missing or either 'from' or 'to' is empty/null, returns true.
|
||||
* - Supports windows that cross midnight (e.g., from '23:00' to '06:00').
|
||||
*
|
||||
* Time parsing is based on the local timezone of the running process.
|
||||
*
|
||||
* @param {{workingHours?: {from?: string|null, to?: string|null}}} config - Configuration object containing working hours in 'HH:mm' format.
|
||||
* @param {number} now - Epoch milliseconds to evaluate.
|
||||
* @returns {boolean} True when execution is allowed at 'now'.
|
||||
* @example
|
||||
* // Same-day window
|
||||
* duringWorkingHoursOrNotSet({ workingHours: { from: '08:00', to: '17:00' } }, someTime);
|
||||
* @example
|
||||
* // Window crossing midnight
|
||||
* // For { from: '05:00', to: '00:30' } → 23:00 => true, 01:00 => false, 06:00 => true
|
||||
* duringWorkingHoursOrNotSet({ workingHours: { from: '05:00', to: '00:30' } }, Date.now());
|
||||
*/
|
||||
function duringWorkingHoursOrNotSet(config, now) {
|
||||
const { workingHours } = config;
|
||||
@@ -100,7 +133,20 @@ function duringWorkingHoursOrNotSet(config, now) {
|
||||
}
|
||||
const toDate = timeStringToMs(workingHours.to, now);
|
||||
const fromDate = timeStringToMs(workingHours.from, now);
|
||||
return fromDate <= now && toDate >= now;
|
||||
|
||||
// If parsing fails (e.g., malformed time), be lenient and allow.
|
||||
if (isNaN(toDate) || isNaN(fromDate)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (toDate >= fromDate) {
|
||||
// Same-day window (e.g., 08:00 - 17:00)
|
||||
return now >= fromDate && now <= toDate;
|
||||
}
|
||||
|
||||
// Window crosses midnight (e.g., 05:00 -> 00:30 next day)
|
||||
// Accept if we are after 'from' today OR before 'to' today (which represents next day's cutoff).
|
||||
return now >= fromDate || now <= toDate;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -134,6 +180,23 @@ function buildHash(...inputs) {
|
||||
*/
|
||||
let config = {};
|
||||
|
||||
/**
|
||||
* If the config exists, but cannot be accessed, we quit Fredy as something is fishy here.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
export async function checkIfConfigIsAccessible() {
|
||||
const path = new URL('../conf/config.json', import.meta.url);
|
||||
try {
|
||||
if (!fs.existsSync(path)) {
|
||||
return true;
|
||||
}
|
||||
fs.accessSync(path, fs.constants.R_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read config JSON from disk (conf/config.json) and parse it.
|
||||
* @returns {Promise<any>} Parsed configuration object.
|
||||
@@ -196,22 +259,56 @@ const normalizeImageUrl = (url) => {
|
||||
return u;
|
||||
};
|
||||
|
||||
/**
|
||||
* returns Fredy's version
|
||||
* @returns {Promise<*|string>}
|
||||
*/
|
||||
async function getPackageVersion() {
|
||||
try {
|
||||
const packagePath = await packageUp();
|
||||
const packageJson = readFileSync(packagePath, 'utf8');
|
||||
const json = JSON.parse(packageJson);
|
||||
return json.version;
|
||||
} catch (error) {
|
||||
logger.error('Error reading version from package.json', error);
|
||||
}
|
||||
return 'N/A';
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep helper
|
||||
* @param {number} ms milliseconds to wait
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a random integer between min and max (inclusive).
|
||||
* @param {number} min - Minimum integer value.
|
||||
* @param {number} max - Maximum integer value.
|
||||
* @returns {number} A random integer N where min <= N <= max.
|
||||
*/
|
||||
function randomBetween(min, max) {
|
||||
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
}
|
||||
|
||||
// Call refreshConfig() from the application entrypoint during startup to populate config.
|
||||
await refreshConfig();
|
||||
|
||||
export { isOneOf };
|
||||
export { normalizeImageUrl };
|
||||
export { inDevMode };
|
||||
export { nullOrEmpty };
|
||||
export { duringWorkingHoursOrNotSet };
|
||||
export { getDirName };
|
||||
export { config };
|
||||
export { buildHash };
|
||||
export default {
|
||||
export {
|
||||
isOneOf,
|
||||
normalizeImageUrl,
|
||||
inDevMode,
|
||||
nullOrEmpty,
|
||||
duringWorkingHoursOrNotSet,
|
||||
getDirName,
|
||||
sleep,
|
||||
randomBetween,
|
||||
config,
|
||||
buildHash,
|
||||
getPackageVersion,
|
||||
toJson,
|
||||
fromJson,
|
||||
};
|
||||
|
||||
49
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fredy",
|
||||
"version": "12.0.0",
|
||||
"version": "14.1.0",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"prepare": "husky",
|
||||
@@ -14,8 +14,8 @@
|
||||
"test": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 test/**/*.test.js",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "yarn lint --fix",
|
||||
"migratedb": "node db/migrations/migrate.js",
|
||||
"migratedb:overwrite": "x-var MIGRATION_ALLOW_CHECKSUM_UPDATE=true node db/migrations/migrate.js"
|
||||
"migratedb": "node lib/services/storage/migrations/migrate.js",
|
||||
"migratedb:overwrite": "x-var MIGRATION_ALLOW_CHECKSUM_UPDATE=true node lib/services/storage/migrations/migrate.js"
|
||||
},
|
||||
"type": "module",
|
||||
"lint-staged": {
|
||||
@@ -56,61 +56,58 @@
|
||||
"Firefox ESR"
|
||||
],
|
||||
"dependencies": {
|
||||
"@douyinfe/semi-icons": "^2.86.0",
|
||||
"@douyinfe/semi-ui": "2.86.0",
|
||||
"@rematch/core": "2.2.0",
|
||||
"@rematch/loading": "2.1.2",
|
||||
"@sendgrid/mail": "8.1.5",
|
||||
"@visactor/react-vchart": "^2.0.4",
|
||||
"@visactor/vchart": "^2.0.4",
|
||||
"@sendgrid/mail": "8.1.6",
|
||||
"@visactor/react-vchart": "^2.0.5",
|
||||
"@visactor/vchart": "^2.0.5",
|
||||
"@visactor/vchart-semi-theme": "^1.12.2",
|
||||
"@vitejs/plugin-react": "5.0.3",
|
||||
"better-sqlite3": "^12.2.0",
|
||||
"@vitejs/plugin-react": "5.0.4",
|
||||
"better-sqlite3": "^12.4.1",
|
||||
"body-parser": "2.2.0",
|
||||
"cheerio": "^1.1.2",
|
||||
"cookie-session": "2.1.1",
|
||||
"handlebars": "4.7.8",
|
||||
"lodash": "4.17.21",
|
||||
"markdown": "^0.5.0",
|
||||
"nanoid": "5.1.5",
|
||||
"nanoid": "5.1.6",
|
||||
"node-cron": "^4.2.1",
|
||||
"node-fetch": "3.3.2",
|
||||
"node-mailjet": "6.0.9",
|
||||
"p-throttle": "^8.0.0",
|
||||
"package-up": "^5.0.0",
|
||||
"puppeteer": "^24.22.0",
|
||||
"puppeteer": "^24.23.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"query-string": "9.3.0",
|
||||
"query-string": "9.3.1",
|
||||
"react": "18.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
"react-redux": "9.2.0",
|
||||
"react-router": "7.9.1",
|
||||
"react-router-dom": "7.9.1",
|
||||
"redux": "5.0.1",
|
||||
"redux-thunk": "3.1.0",
|
||||
"react-router": "7.9.3",
|
||||
"react-router-dom": "7.9.3",
|
||||
"restana": "5.1.0",
|
||||
"semver": "^7.7.2",
|
||||
"serve-static": "2.2.0",
|
||||
"slack": "11.0.2",
|
||||
"vite": "7.1.6",
|
||||
"x-var": "^3.0.1"
|
||||
"vite": "7.1.9",
|
||||
"x-var": "^3.0.1",
|
||||
"zustand": "^5.0.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "7.28.4",
|
||||
"@babel/eslint-parser": "7.28.4",
|
||||
"@babel/preset-env": "7.28.3",
|
||||
"@babel/preset-react": "7.27.1",
|
||||
"chai": "6.0.1",
|
||||
"eslint": "9.35.0",
|
||||
"chai": "6.2.0",
|
||||
"eslint": "9.36.0",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-plugin-react": "7.37.5",
|
||||
"esmock": "2.7.3",
|
||||
"history": "5.3.0",
|
||||
"husky": "9.1.7",
|
||||
"less": "4.4.1",
|
||||
"lint-staged": "16.1.6",
|
||||
"mocha": "11.7.2",
|
||||
"lint-staged": "16.2.3",
|
||||
"mocha": "11.7.4",
|
||||
"nodemon": "^3.1.10",
|
||||
"prettier": "3.6.2",
|
||||
"redux-logger": "3.0.6"
|
||||
"prettier": "3.6.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ Challenges:
|
||||
_Returns the total number of listings for the given query._
|
||||
|
||||
```
|
||||
curl -H "User-Agent: ImmoScout24_1410_30_._" \
|
||||
curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
|
||||
-H "Accept: application/json" \
|
||||
"https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
|
||||
```
|
||||
@@ -63,7 +63,7 @@ _The body is json encoded and contains data specifying additional results (adver
|
||||
```
|
||||
curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
|
||||
-H "Connection: keep-alive" \
|
||||
-H "User-Agent: ImmoScout24_1410_30_._" \
|
||||
-H "User-Agent: ImmoScout_27.3_26.0_._" \
|
||||
-H "Accept: application/json" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"supportedResultListType":[],"userData":{}}'
|
||||
@@ -78,7 +78,7 @@ curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calc
|
||||
The response contains additional details not included in the listing response.
|
||||
|
||||
```
|
||||
curl -H "User-Agent: ImmoScout24_1410_30_._" \
|
||||
curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
|
||||
-H "Accept: application/json" \
|
||||
"https://api.mobile.immobilienscout24.de/expose/158382494"
|
||||
```
|
||||
|
||||
53
test/FredyRuntime/FredyRuntime.test.js
Normal file
@@ -0,0 +1,53 @@
|
||||
import { expect } from 'chai';
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { mockFredy } from '../utils.js';
|
||||
|
||||
describe('FredyRuntime', () => {
|
||||
afterEach(() => {
|
||||
similarityCache.invalidateAllForTest();
|
||||
});
|
||||
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
|
||||
describe('_filterBySimilarListings', () => {
|
||||
let fredyRuntime;
|
||||
|
||||
beforeEach(async () => {
|
||||
const FredyRuntime = await mockFredy();
|
||||
fredyRuntime = new FredyRuntime({}, null, 'dummy-provider', 'dummy-job', similarityCache);
|
||||
});
|
||||
|
||||
it('should filter out listings with similar title and address already in cache', () => {
|
||||
similarityCache.addCacheEntry('Penthouse', 'Mustermann Straße 1');
|
||||
|
||||
const listings = [
|
||||
{ id: '1', title: 'Penthouse', address: 'Mustermann Straße 1' },
|
||||
{ id: '2', title: 'Nice apartment', address: 'Mustermann Straße 15' },
|
||||
];
|
||||
|
||||
const result = fredyRuntime._filterBySimilarListings(listings);
|
||||
|
||||
expect(result).to.have.length(1);
|
||||
expect(result[0].id).to.equal('2');
|
||||
expect(result[0].title).to.equal('Nice apartment');
|
||||
|
||||
expect(similarityCache.hasSimilarEntries('Nice apartment', 'Mustermann Straße 15')).to.be.true;
|
||||
});
|
||||
|
||||
it('should handle listings with null or undefined address', () => {
|
||||
const listings = [
|
||||
{ id: '1', title: 'Penthouse', address: null },
|
||||
{ id: '2', title: 'Nice apartment', address: undefined },
|
||||
];
|
||||
|
||||
const result = fredyRuntime._filterBySimilarListings(listings);
|
||||
|
||||
expect(result).to.have.length(2);
|
||||
|
||||
expect(similarityCache.hasSimilarEntries('Penthouse', null)).to.be.true;
|
||||
expect(similarityCache.hasSimilarEntries('Nice apartment', undefined)).to.be.true;
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -307,7 +307,7 @@ describe('db/migrations/migrate.js - runMigrations', () => {
|
||||
const loggerPath = path.join(ROOT, 'lib', 'services', 'logger.js');
|
||||
|
||||
const mod = await esmock(
|
||||
'../../../db/migrations/migrate.js',
|
||||
'../../../lib/services/storage/migrations/migrate.js',
|
||||
{},
|
||||
{
|
||||
fs: fsMock,
|
||||
|
||||
@@ -8,31 +8,30 @@ describe('#immonet testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
provider.init(providerConfig.immonet, [], []);
|
||||
|
||||
it('should test immonet provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immonet');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
provider.init(providerConfig.immonet, [], []);
|
||||
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immonet');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,33 +8,32 @@ describe('#immowelt testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
|
||||
it('should test immowelt provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.immowelt, [], []);
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immowelt');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).that.does.include('m²');
|
||||
}
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immowelt.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immowelt');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).that.does.include('m²');
|
||||
}
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immowelt.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
37
test/provider/mcMakler.test.js
Normal file
@@ -0,0 +1,37 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/mcMakler.js';
|
||||
|
||||
describe('#mcMakler testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
|
||||
it('should test mcMakler provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.mcMakler, []);
|
||||
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'mcMakler', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('mcMakler');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
});
|
||||
});
|
||||
43
test/provider/regionalimmobilien24.test.js
Normal file
@@ -0,0 +1,43 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/regionalimmobilien24.js';
|
||||
|
||||
describe('#regionalimmobilien24 testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
|
||||
it('should test regionalimmobilien24 provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.regionalimmobilien24, []);
|
||||
|
||||
const fredy = new Fredy(
|
||||
provider.config,
|
||||
null,
|
||||
provider.metaInformation.id,
|
||||
'regionalimmobilien24',
|
||||
similarityCache,
|
||||
);
|
||||
const listing = await fredy.execute();
|
||||
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('regionalimmobilien24');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
});
|
||||
});
|
||||
37
test/provider/sparkasse.test.js
Normal file
@@ -0,0 +1,37 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect } from 'chai';
|
||||
import * as provider from '../../lib/provider/sparkasse.js';
|
||||
|
||||
describe('#sparkasse testsuite()', () => {
|
||||
after(() => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
|
||||
it('should test sparkasse provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.sparkasse, []);
|
||||
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'sparkasse', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('sparkasse');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -28,10 +28,22 @@
|
||||
"url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
|
||||
"enabled": true
|
||||
},
|
||||
"mcMakler": {
|
||||
"url": "https://www.mcmakler.de/immobilien/results?placeId=62649&search=Leipzig%252C+Sachsen&propertyTypes=APARTMENT&page=0",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"regionalimmobilien24": {
|
||||
"url": "https://www.regionalimmobilien24.de/rostock/rostock/kaufen/haus/-/-/-/?rd=5",
|
||||
"enabled": true
|
||||
},
|
||||
"sparkasse": {
|
||||
"url": "https://immobilien.sparkasse.de/immobilien/treffer?marketingType=buy&objectType=flat&perimeter=10&usageType=residential&zipCityEstateId=62782__Hamburg",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
|
||||
"enabled": true
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import utils from '../../lib/utils.js';
|
||||
import { isOneOf, duringWorkingHoursOrNotSet } from '../../lib/utils.js';
|
||||
import assert from 'assert';
|
||||
import { expect } from 'chai';
|
||||
|
||||
@@ -8,30 +8,45 @@ const fakeWorkingHoursConfig = (from, to) => ({
|
||||
from,
|
||||
},
|
||||
});
|
||||
|
||||
describe('utils', () => {
|
||||
describe('#isOneOf()', () => {
|
||||
it('should be false', () => {
|
||||
assert.equal(utils.isOneOf('bla', ['blub']), false);
|
||||
assert.equal(isOneOf('bla', ['blub']), false);
|
||||
});
|
||||
it('should be true', () => {
|
||||
assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
|
||||
assert.equal(isOneOf('bla blub blubber', ['bla']), true);
|
||||
});
|
||||
});
|
||||
describe('#duringWorkingHoursOrNotSet()', () => {
|
||||
it('should be false', () => {
|
||||
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
|
||||
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
|
||||
});
|
||||
it('should be true', () => {
|
||||
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
|
||||
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
|
||||
});
|
||||
it('should be true if nothing set', () => {
|
||||
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
|
||||
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
|
||||
});
|
||||
it('should be true if only to is set', () => {
|
||||
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
|
||||
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
|
||||
});
|
||||
it('should be true if only from is set', () => {
|
||||
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
|
||||
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
|
||||
});
|
||||
it('should handle working hours that cross midnight (e.g., 05:00 → 00:30)', () => {
|
||||
const cfg = fakeWorkingHoursConfig('05:00', '00:30');
|
||||
const mkTs = (h, m = 0) => {
|
||||
const d = new Date();
|
||||
d.setHours(h);
|
||||
d.setMinutes(m);
|
||||
d.setSeconds(0);
|
||||
d.setMilliseconds(0);
|
||||
return d.getTime();
|
||||
};
|
||||
expect(duringWorkingHoursOrNotSet(cfg, mkTs(23, 0))).to.be.true; // 23:00 => within window
|
||||
expect(duringWorkingHoursOrNotSet(cfg, mkTs(1, 0))).to.be.false; // 01:00 => outside window
|
||||
expect(duringWorkingHoursOrNotSet(cfg, mkTs(6, 0))).to.be.true; // 06:00 => within window
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,7 +53,7 @@ describe('#immoscout-mobile URL conversion', () => {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': 'ImmoScout24_1410_30_._',
|
||||
'User-Agent': 'ImmoScout_27.3_26.0_._',
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
|
||||
171
ui/src/App.jsx
@@ -6,34 +6,40 @@ import GeneralSettings from './views/generalSettings/GeneralSettings';
|
||||
import JobMutation from './views/jobs/mutation/JobMutation';
|
||||
import UserMutator from './views/user/mutation/UserMutator';
|
||||
import JobInsight from './views/jobs/insights/JobInsight.jsx';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import { useActions, useSelector } from './services/state/store';
|
||||
import { Routes, Route, Navigate } from 'react-router-dom';
|
||||
import Logout from './components/logout/Logout';
|
||||
import Logo from './components/logo/Logo';
|
||||
import Menu from './components/menu/Menu';
|
||||
import Login from './views/login/Login';
|
||||
import Users from './views/user/Users';
|
||||
import Jobs from './views/jobs/Jobs';
|
||||
|
||||
import './App.less';
|
||||
import TrackingModal from './components/tracking/TrackingModal.jsx';
|
||||
import { Banner } from '@douyinfe/semi-ui';
|
||||
import { Banner, Divider } from '@douyinfe/semi-ui';
|
||||
import VersionBanner from './components/version/VersionBanner.jsx';
|
||||
import Listings from './views/listings/Listings.jsx';
|
||||
import Navigation from './components/navigation/Navigation.jsx';
|
||||
import { Layout } from '@douyinfe/semi-ui';
|
||||
import FredyFooter from './components/footer/FredyFooter.jsx';
|
||||
import ProcessingTimes from './views/jobs/ProcessingTimes.jsx';
|
||||
|
||||
export default function FredyApp() {
|
||||
const dispatch = useDispatch();
|
||||
const actions = useActions();
|
||||
const [loading, setLoading] = React.useState(true);
|
||||
const currentUser = useSelector((state) => state.user.currentUser);
|
||||
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
|
||||
const settings = useSelector((state) => state.generalSettings.settings);
|
||||
const processingTimes = useSelector((state) => state.jobs.processingTimes);
|
||||
|
||||
useEffect(() => {
|
||||
async function init() {
|
||||
await dispatch.user.getCurrentUser();
|
||||
await actions.user.getCurrentUser();
|
||||
if (!needsLogin()) {
|
||||
await dispatch.provider.getProvider();
|
||||
await dispatch.jobs.getJobs();
|
||||
await dispatch.jobs.getProcessingTimes();
|
||||
await dispatch.notificationAdapter.getAdapter();
|
||||
await dispatch.generalSettings.getGeneralSettings();
|
||||
await actions.provider.getProvider();
|
||||
await actions.jobs.getJobs();
|
||||
await actions.jobs.getProcessingTimes();
|
||||
await actions.notificationAdapter.getAdapter();
|
||||
await actions.generalSettings.getGeneralSettings();
|
||||
await actions.versionUpdate.getVersionUpdate();
|
||||
}
|
||||
setLoading(false);
|
||||
}
|
||||
@@ -46,81 +52,88 @@ export default function FredyApp() {
|
||||
};
|
||||
|
||||
const isAdmin = () => currentUser != null && currentUser.isAdmin;
|
||||
const { Footer, Sider, Content } = Layout;
|
||||
|
||||
const login = () => (
|
||||
return loading ? null : needsLogin() ? (
|
||||
<Routes>
|
||||
<Route path="/login" element={<Login />} />
|
||||
<Route path="*" element={<Navigate to="/login" replace />} />
|
||||
</Routes>
|
||||
);
|
||||
|
||||
return loading ? null : needsLogin() ? (
|
||||
login()
|
||||
) : (
|
||||
<div className="app">
|
||||
<div className="app__container">
|
||||
<Logout />
|
||||
<Logo width={190} white />
|
||||
<Menu isAdmin={isAdmin()} />
|
||||
<Layout className="app">
|
||||
<Layout className="app">
|
||||
<Sider>
|
||||
<Navigation isAdmin={isAdmin()} />
|
||||
</Sider>
|
||||
<Content>
|
||||
{versionUpdate?.newVersion && <VersionBanner />}
|
||||
{settings.demoMode && (
|
||||
<>
|
||||
<Banner
|
||||
fullMode={true}
|
||||
type="info"
|
||||
bordered
|
||||
closeIcon={null}
|
||||
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
|
||||
/>
|
||||
<br />
|
||||
</>
|
||||
)}
|
||||
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
|
||||
{processingTimes != null && <ProcessingTimes processingTimes={processingTimes} />}
|
||||
<Divider />
|
||||
<div className="app__content">
|
||||
<Routes>
|
||||
<Route path="/403" element={<InsufficientPermission />} />
|
||||
<Route path="/jobs/new" element={<JobMutation />} />
|
||||
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
|
||||
<Route path="/jobs/insights/:jobId" element={<JobInsight />} />
|
||||
<Route path="/jobs" element={<Jobs />} />
|
||||
<Route path="/listings" element={<Listings />} />
|
||||
|
||||
{settings.demoMode && (
|
||||
<>
|
||||
<Banner
|
||||
fullMode={true}
|
||||
type="info"
|
||||
bordered
|
||||
closeIcon={null}
|
||||
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
|
||||
/>
|
||||
<br />
|
||||
</>
|
||||
)}
|
||||
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
|
||||
<Routes>
|
||||
<Route path="/403" element={<InsufficientPermission />} />
|
||||
<Route path="/jobs/new" element={<JobMutation />} />
|
||||
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
|
||||
<Route path="/jobs/insights/:jobId" element={<JobInsight />} />
|
||||
<Route path="/jobs" element={<Jobs />} />
|
||||
{/* Permission-aware routes */}
|
||||
<Route
|
||||
path="/users/new"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<UserMutator />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/users/edit/:userId"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<UserMutator />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/users"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<Users />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/generalSettings"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<GeneralSettings />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
|
||||
{/* Permission-aware routes */}
|
||||
<Route
|
||||
path="/users/new"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<UserMutator />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/users/edit/:userId"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<UserMutator />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/users"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<Users />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="/generalSettings"
|
||||
element={
|
||||
<PermissionAwareRoute currentUser={currentUser}>
|
||||
<GeneralSettings />
|
||||
</PermissionAwareRoute>
|
||||
}
|
||||
/>
|
||||
|
||||
<Route path="/" element={<Navigate to="/jobs" replace />} />
|
||||
</Routes>
|
||||
</div>
|
||||
</div>
|
||||
<Route path="/" element={<Navigate to="/jobs" replace />} />
|
||||
</Routes>
|
||||
</div>
|
||||
</Content>
|
||||
</Layout>
|
||||
<Footer>
|
||||
<FredyFooter />
|
||||
</Footer>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
.app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
|
||||
&__container {
|
||||
padding: 1rem 1rem;
|
||||
color: var(--semi-color-text-0);
|
||||
background-color: #232429;
|
||||
&__content {
|
||||
margin: 1rem;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import React from 'react';
|
||||
|
||||
import { reduxStore } from './services/rematch/store';
|
||||
import { HashRouter } from 'react-router-dom';
|
||||
import { Provider } from 'react-redux';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
|
||||
import { LocaleProvider } from '@douyinfe/semi-ui';
|
||||
@@ -18,11 +16,9 @@ initVChartSemiTheme({
|
||||
});
|
||||
|
||||
root.render(
|
||||
<Provider store={reduxStore}>
|
||||
<HashRouter>
|
||||
<LocaleProvider locale={en_US}>
|
||||
<App />
|
||||
</LocaleProvider>
|
||||
</HashRouter>
|
||||
</Provider>,
|
||||
<HashRouter>
|
||||
<LocaleProvider locale={en_US}>
|
||||
<App />
|
||||
</LocaleProvider>
|
||||
</HashRouter>,
|
||||
);
|
||||
|
||||
BIN
ui/src/assets/no_image.jpg
Normal file
|
After Width: | Height: | Size: 242 KiB |
19
ui/src/components/footer/FredyFooter.jsx
Normal file
@@ -0,0 +1,19 @@
|
||||
import React from 'react';
|
||||
import './FredyFooter.less';
|
||||
import { useSelector } from '../../services/state/store.js';
|
||||
import { Typography } from '@douyinfe/semi-ui';
|
||||
|
||||
export default function FredyFooter() {
|
||||
const { Text } = Typography;
|
||||
const version = useSelector((state) => state.versionUpdate.versionUpdate);
|
||||
return (
|
||||
<div className="fredyFooter">
|
||||
<div className="fredyFooter__version">
|
||||
<Text type="tertiary">Fredy V{version?.localFredyVersion || 'N/A'}</Text>
|
||||
</div>
|
||||
<div className="fredyFooter__copyRight">
|
||||
<Text link={{ href: 'https://github.com/orangecoding', target: '_blank' }}>Made with ❤️</Text>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
18
ui/src/components/footer/FredyFooter.less
Normal file
@@ -0,0 +1,18 @@
|
||||
.fredyFooter {
|
||||
background:rgb(53, 54, 60);
|
||||
color: white;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
height: 1.7rem;
|
||||
|
||||
&__version {
|
||||
padding-left: .5rem;
|
||||
font-size: small;
|
||||
|
||||
}
|
||||
&__copyRight {
|
||||
padding-right: 1rem;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -5,5 +5,5 @@ import logoWhite from '../../assets/logo_white.png';
|
||||
import './Logo.less';
|
||||
|
||||
export default function Logo({ width = 350, white = false } = {}) {
|
||||
return <img src={white ? logoWhite : logo} width={width} className="logo" />;
|
||||
return <img src={white ? logoWhite : logo} width={width} className="logo" alt="Fredy Logo" />;
|
||||
}
|
||||
|
||||
@@ -2,19 +2,22 @@ import React from 'react';
|
||||
import { Button } from '@douyinfe/semi-ui';
|
||||
import { xhrPost } from '../../services/xhr';
|
||||
import { IconUser } from '@douyinfe/semi-icons';
|
||||
const Logout = function Logout() {
|
||||
|
||||
const Logout = function Logout({ text }) {
|
||||
return (
|
||||
<Button
|
||||
icon={<IconUser />}
|
||||
type="danger"
|
||||
theme="solid"
|
||||
onClick={async () => {
|
||||
await xhrPost('/api/login/logout');
|
||||
location.reload();
|
||||
}}
|
||||
>
|
||||
Logout
|
||||
</Button>
|
||||
<div>
|
||||
<Button
|
||||
icon={<IconUser />}
|
||||
type="danger"
|
||||
theme="solid"
|
||||
onClick={async () => {
|
||||
await xhrPost('/api/login/logout');
|
||||
location.reload();
|
||||
}}
|
||||
>
|
||||
{text && 'Logout'}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
import React from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import { Tabs, TabPane } from '@douyinfe/semi-ui';
|
||||
|
||||
import { useLocation } from 'react-router-dom';
|
||||
import { IconUser, IconTerminal, IconSetting } from '@douyinfe/semi-icons';
|
||||
import './Menu.less';
|
||||
|
||||
function parsePathName(name) {
|
||||
const split = name.split('/').filter((s) => s.length !== 0);
|
||||
return '/' + split[0];
|
||||
}
|
||||
|
||||
const TopMenu = function TopMenu({ isAdmin }) {
|
||||
const navigate = useNavigate();
|
||||
const location = useLocation();
|
||||
return (
|
||||
<Tabs className="menu" type="line" activeKey={parsePathName(location.pathname)} onTabClick={(key) => navigate(key)}>
|
||||
<TabPane
|
||||
itemKey="/jobs"
|
||||
tab={
|
||||
<span>
|
||||
<IconTerminal />
|
||||
Jobs
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
|
||||
{isAdmin && (
|
||||
<TabPane
|
||||
itemKey="/users"
|
||||
tab={
|
||||
<span>
|
||||
<IconUser />
|
||||
User
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{isAdmin && (
|
||||
<TabPane
|
||||
itemKey="/generalSettings"
|
||||
tab={
|
||||
<span>
|
||||
<IconSetting />
|
||||
General
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</Tabs>
|
||||
);
|
||||
};
|
||||
|
||||
export default TopMenu;
|
||||
@@ -1,3 +0,0 @@
|
||||
.menu {
|
||||
margin-top: 3rem;
|
||||
}
|
||||
9
ui/src/components/navigation/Navigate.less
Normal file
@@ -0,0 +1,9 @@
|
||||
.navigate {
|
||||
&__logout_Button {
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
display: flex;
|
||||
|
||||
}
|
||||
}
|
||||
50
ui/src/components/navigation/Navigation.jsx
Normal file
@@ -0,0 +1,50 @@
|
||||
import React from 'react';
|
||||
import { Nav } from '@douyinfe/semi-ui';
|
||||
import { IconUser, IconStar, IconSetting, IconTerminal } from '@douyinfe/semi-icons';
|
||||
import logoWhite from '../../assets/logo_white.png';
|
||||
import Logout from '../logout/Logout.jsx';
|
||||
import { useLocation, useNavigate } from 'react-router-dom';
|
||||
|
||||
import './Navigate.less';
|
||||
import { useScreenWidth } from '../../hooks/screenWidth.js';
|
||||
|
||||
export default function Navigation({ isAdmin }) {
|
||||
const navigate = useNavigate();
|
||||
const location = useLocation();
|
||||
|
||||
const width = useScreenWidth();
|
||||
const collapsed = width <= 850;
|
||||
|
||||
const items = [
|
||||
{ itemKey: '/jobs', text: 'Jobs', icon: <IconTerminal /> },
|
||||
{ itemKey: '/listings', text: 'Found Listings', icon: <IconStar /> },
|
||||
];
|
||||
|
||||
if (isAdmin) {
|
||||
items.push({ itemKey: '/users', text: 'User Management', icon: <IconUser /> });
|
||||
items.push({ itemKey: '/generalSettings', text: 'Settings', icon: <IconSetting /> });
|
||||
}
|
||||
|
||||
function parsePathName(name) {
|
||||
const split = name.split('/').filter((s) => s.length !== 0);
|
||||
return '/' + split[0];
|
||||
}
|
||||
|
||||
return (
|
||||
<Nav
|
||||
style={{ height: '100%', width: collapsed ? '' : '13rem' }}
|
||||
items={items}
|
||||
isCollapsed={collapsed}
|
||||
selectedKeys={[parsePathName(location.pathname)]}
|
||||
onSelect={(key) => {
|
||||
navigate(key.itemKey);
|
||||
}}
|
||||
header={<img src={logoWhite} width="180" alt="Fredy Logo" />}
|
||||
footer={
|
||||
<div className="navigate__logout_Button">
|
||||
<Logout text={!collapsed} />
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -8,6 +8,7 @@ export const SegmentPart = ({ name, Icon = null, children, helpText }) => {
|
||||
|
||||
return (
|
||||
<Card
|
||||
className="segmentParts"
|
||||
title={
|
||||
<Meta title={name} description={helpText} avatar={Icon == null ? null : <Icon size="extra-extra-small" />} />
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
.segmentParts {
|
||||
border: 1px solid #323232 !important;
|
||||
border-radius: 5px !important;
|
||||
color: rgba(var(--semi-grey-8), 1);
|
||||
background: rgb(53, 54, 60);
|
||||
margin: 2rem;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import React from 'react';
|
||||
|
||||
import { Button, Empty, Table, Switch } from '@douyinfe/semi-ui';
|
||||
import { IconDelete, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
|
||||
import { Button, Empty, Table, Switch, Popover } from '@douyinfe/semi-ui';
|
||||
import { IconDelete, IconDescend2, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
|
||||
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
|
||||
|
||||
import './JobTable.less';
|
||||
@@ -10,11 +10,20 @@ const empty = (
|
||||
<Empty
|
||||
image={<IllustrationNoResult />}
|
||||
darkModeImage={<IllustrationNoResultDark />}
|
||||
description={'No jobs available.'}
|
||||
description="No jobs available. Why don't you create one? ;)"
|
||||
/>
|
||||
);
|
||||
|
||||
export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight } = {}) {
|
||||
const getPopoverContent = (text) => <article className="jobPopoverContent">{text}</article>;
|
||||
|
||||
export default function JobTable({
|
||||
jobs = {},
|
||||
onJobRemoval,
|
||||
onJobStatusChanged,
|
||||
onJobEdit,
|
||||
onJobInsight,
|
||||
onListingRemoval,
|
||||
} = {}) {
|
||||
return (
|
||||
<Table
|
||||
pagination={false}
|
||||
@@ -32,7 +41,7 @@ export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged,
|
||||
dataIndex: 'name',
|
||||
},
|
||||
{
|
||||
title: 'Findings',
|
||||
title: 'Listings',
|
||||
dataIndex: 'numberOfFoundListings',
|
||||
render: (value) => {
|
||||
return value || 0;
|
||||
@@ -58,9 +67,18 @@ export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged,
|
||||
render: (_, job) => {
|
||||
return (
|
||||
<div className="interactions">
|
||||
<Button type="primary" icon={<IconHistogram />} onClick={() => onJobInsight(job.id)} />
|
||||
<Button type="secondary" icon={<IconEdit />} onClick={() => onJobEdit(job.id)} />
|
||||
<Button type="danger" icon={<IconDelete />} onClick={() => onJobRemoval(job.id)} />
|
||||
<Popover content={getPopoverContent('Job Insights')}>
|
||||
<Button type="primary" icon={<IconHistogram />} onClick={() => onJobInsight(job.id)} />
|
||||
</Popover>
|
||||
<Popover content={getPopoverContent('Edit a Job')}>
|
||||
<Button type="secondary" icon={<IconEdit />} onClick={() => onJobEdit(job.id)} />
|
||||
</Popover>
|
||||
<Popover content={getPopoverContent('Delete all found Listings of this Job')}>
|
||||
<Button type="danger" icon={<IconDescend2 />} onClick={() => onListingRemoval(job.id)} />
|
||||
</Popover>
|
||||
<Popover content={getPopoverContent('Delete Job')}>
|
||||
<Button type="danger" icon={<IconDelete />} onClick={() => onJobRemoval(job.id)} />
|
||||
</Popover>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
|
||||
@@ -5,6 +5,11 @@
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.jobPopoverContent {
|
||||
padding: 1rem;
|
||||
color: white;
|
||||
}
|
||||
|
||||
@media (min-width: 768px) {
|
||||
.interactions {
|
||||
flex-direction: initial;
|
||||
|
||||
45
ui/src/components/table/listings/ListingsFilter.jsx
Normal file
@@ -0,0 +1,45 @@
|
||||
import { Card, Checkbox, Descriptions, Divider, Select } from '@douyinfe/semi-ui';
|
||||
import React from 'react';
|
||||
import { useSelector } from '../../../services/state/store.js';
|
||||
import { Typography } from '@douyinfe/semi-ui';
|
||||
|
||||
import './ListingsFilter.less';
|
||||
|
||||
export default function ListingsFilter({ onWatchListFilter, onActivityFilter, onJobNameFilter, onProviderFilter }) {
|
||||
const jobs = useSelector((state) => state.jobs.jobs);
|
||||
const provider = useSelector((state) => state.provider);
|
||||
const { Title } = Typography;
|
||||
return (
|
||||
<Card className="listingsFilter">
|
||||
<Title heading={6}>Filter by:</Title>
|
||||
<Divider />
|
||||
<br />
|
||||
<Descriptions row>
|
||||
<Descriptions.Item itemKey="Watch List">
|
||||
<Checkbox onChange={(e) => onWatchListFilter(e.target.checked)}>Only Watch List</Checkbox>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Activity status">
|
||||
<Checkbox onChange={(e) => onActivityFilter(e.target.checked)}>Only Active Listings</Checkbox>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Job Name">
|
||||
<Select showClear placeholder="Select Job to Filter" onChange={(val) => onJobNameFilter(val)}>
|
||||
{jobs != null &&
|
||||
jobs.length > 0 &&
|
||||
jobs.map((job) => {
|
||||
return <Select.Option value={job.id}>{job.name}</Select.Option>;
|
||||
})}
|
||||
</Select>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Provider">
|
||||
<Select showClear placeholder="Select Provider to Filter" onChange={(val) => onProviderFilter(val)}>
|
||||
{provider != null &&
|
||||
provider.length > 0 &&
|
||||
provider.map((prov) => {
|
||||
return <Select.Option value={prov.id}>{prov.name}</Select.Option>;
|
||||
})}
|
||||
</Select>
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
4
ui/src/components/table/listings/ListingsFilter.less
Normal file
@@ -0,0 +1,4 @@
|
||||
.listingsFilter {
|
||||
margin-bottom: 1rem;
|
||||
background: rgb(53, 54, 60);
|
||||
}
|
||||
289
ui/src/components/table/listings/ListingsTable.jsx
Normal file
@@ -0,0 +1,289 @@
|
||||
import React, { useState, useEffect, useMemo } from 'react';
|
||||
import { Table, Popover, Input, Descriptions, Tag, Image, Empty, Button, Toast, Divider } from '@douyinfe/semi-ui';
|
||||
import { useActions, useSelector } from '../../../services/state/store.js';
|
||||
import { IconClose, IconDelete, IconSearch, IconStar, IconStarStroked, IconTick } from '@douyinfe/semi-icons';
|
||||
import * as timeService from '../../../services/time/timeService.js';
|
||||
import debounce from 'lodash/debounce';
|
||||
import no_image from '../../../assets/no_image.jpg';
|
||||
|
||||
import './ListingsTable.less';
|
||||
import { format } from '../../../services/time/timeService.js';
|
||||
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
|
||||
import { xhrDelete, xhrPost } from '../../../services/xhr.js';
|
||||
import ListingsFilter from './ListingsFilter.jsx';
|
||||
|
||||
const columns = [
|
||||
{
|
||||
title: '#',
|
||||
width: 100,
|
||||
dataIndex: 'isWatched',
|
||||
sorter: true,
|
||||
render: (id, row) => {
|
||||
return (
|
||||
<div>
|
||||
<Popover
|
||||
style={{
|
||||
padding: '.4rem',
|
||||
color: 'var(--semi-color-white)',
|
||||
}}
|
||||
content={row.isWatched === 1 ? 'Unwatch Listing' : 'Watch Listing'}
|
||||
>
|
||||
<Button
|
||||
icon={
|
||||
row.isWatched === 1 ? (
|
||||
<IconStar style={{ color: 'rgba(var(--semi-green-5), 1)' }} />
|
||||
) : (
|
||||
<IconStarStroked />
|
||||
)
|
||||
}
|
||||
theme="borderless"
|
||||
size="small"
|
||||
onClick={async () => {
|
||||
try {
|
||||
await xhrPost('/api/listings/watch', { listingId: row.id });
|
||||
Toast.success(row.isWatched === 1 ? 'Listing removed from Watchlist' : 'Listing added to Watchlist');
|
||||
row.reloadTable();
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
Toast.error('Failed to operate Watchlist');
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Popover>
|
||||
<Divider layout="vertical" margin="4px" />
|
||||
<Popover
|
||||
style={{
|
||||
padding: '.4rem',
|
||||
color: 'var(--semi-color-white)',
|
||||
}}
|
||||
content="Delete Listing"
|
||||
>
|
||||
<Button
|
||||
icon={<IconDelete />}
|
||||
theme="borderless"
|
||||
size="small"
|
||||
type="danger"
|
||||
onClick={async () => {
|
||||
try {
|
||||
await xhrDelete('/api/listings/', { ids: [id] });
|
||||
Toast.success('Listing(s) successfully removed');
|
||||
row.reloadTable();
|
||||
} catch (error) {
|
||||
Toast.error(error);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Popover>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: 'State',
|
||||
dataIndex: 'is_active',
|
||||
width: 84,
|
||||
sorter: true,
|
||||
render: (value) => {
|
||||
return value ? (
|
||||
<div style={{ color: 'rgba(var(--semi-green-6), 1)' }}>
|
||||
<Popover
|
||||
style={{
|
||||
padding: '.4rem',
|
||||
color: 'var(--semi-color-white)',
|
||||
}}
|
||||
content="Listing is still active"
|
||||
>
|
||||
<IconTick />
|
||||
</Popover>
|
||||
</div>
|
||||
) : (
|
||||
<div style={{ color: 'rgba(var(--semi-red-5), 1)' }}>
|
||||
<Popover
|
||||
style={{
|
||||
padding: '.4rem',
|
||||
color: 'var(--semi-color-white)',
|
||||
}}
|
||||
content="Listing is inactive"
|
||||
>
|
||||
<IconClose />
|
||||
</Popover>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: 'Job-Name',
|
||||
sorter: true,
|
||||
ellipsis: true,
|
||||
dataIndex: 'job_name',
|
||||
width: 150,
|
||||
},
|
||||
{
|
||||
title: 'Listing date',
|
||||
width: 130,
|
||||
dataIndex: 'created_at',
|
||||
sorter: true,
|
||||
render: (text) => timeService.format(text, false),
|
||||
},
|
||||
{
|
||||
title: 'Provider',
|
||||
width: 130,
|
||||
dataIndex: 'provider',
|
||||
sorter: true,
|
||||
render: (text) => text.charAt(0).toUpperCase() + text.slice(1),
|
||||
},
|
||||
{
|
||||
title: 'Price',
|
||||
width: 110,
|
||||
dataIndex: 'price',
|
||||
sorter: true,
|
||||
render: (text) => text + ' €',
|
||||
},
|
||||
{
|
||||
title: 'Address',
|
||||
width: 150,
|
||||
dataIndex: 'address',
|
||||
sorter: true,
|
||||
},
|
||||
{
|
||||
title: 'Title',
|
||||
dataIndex: 'title',
|
||||
sorter: true,
|
||||
ellipsis: true,
|
||||
render: (text, row) => {
|
||||
return (
|
||||
<a href={row.url} target="_blank" rel="noopener noreferrer">
|
||||
{text}
|
||||
</a>
|
||||
);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const empty = (
|
||||
<Empty
|
||||
image={<IllustrationNoResult />}
|
||||
darkModeImage={<IllustrationNoResultDark />}
|
||||
description="No listings available."
|
||||
/>
|
||||
);
|
||||
|
||||
export default function ListingsTable() {
|
||||
const tableData = useSelector((state) => state.listingsTable);
|
||||
const actions = useActions();
|
||||
const [page, setPage] = useState(1);
|
||||
const pageSize = 10;
|
||||
const [sortData, setSortData] = useState({});
|
||||
const [freeTextFilter, setFreeTextFilter] = useState(null);
|
||||
const [watchListFilter, setWatchListFilter] = useState(null);
|
||||
const [jobNameFilter, setJobNameFilter] = useState(null);
|
||||
const [activityFilter, setActivityFilter] = useState(null);
|
||||
const [providerFilter, setProviderFilter] = useState(null);
|
||||
|
||||
const handlePageChange = (_page) => {
|
||||
setPage(_page);
|
||||
};
|
||||
|
||||
const loadTable = () => {
|
||||
let sortfield = null;
|
||||
let sortdir = null;
|
||||
|
||||
if (sortData != null && Object.keys(sortData).length > 0) {
|
||||
sortfield = sortData.field;
|
||||
sortdir = sortData.direction;
|
||||
}
|
||||
actions.listingsTable.getListingsTable({
|
||||
page,
|
||||
pageSize,
|
||||
sortfield,
|
||||
sortdir,
|
||||
freeTextFilter,
|
||||
filter: { watchListFilter, jobNameFilter, activityFilter, providerFilter },
|
||||
});
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
loadTable();
|
||||
}, [page, sortData, freeTextFilter, providerFilter, activityFilter, jobNameFilter, watchListFilter]);
|
||||
|
||||
const handleFilterChange = useMemo(() => debounce((value) => setFreeTextFilter(value), 500), []);
|
||||
|
||||
const expandRowRender = (record) => {
|
||||
return (
|
||||
<div className="listingsTable__expanded">
|
||||
<div>
|
||||
{record.image_url == null ? (
|
||||
<Image height={200} src={no_image} />
|
||||
) : (
|
||||
<Image height={200} src={record.image_url} />
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
<Descriptions align="justify">
|
||||
<Descriptions.Item itemKey="Listing still online">
|
||||
<Tag size="small" shape="circle" color={record.is_active ? 'green' : 'red'}>
|
||||
{record.is_active ? 'Yes' : 'No'}
|
||||
</Tag>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Link">
|
||||
<a href={record.link} target="_blank" rel="noreferrer">
|
||||
Link to Listing
|
||||
</a>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Listing date">{format(record.created_at)}</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Price">{record.price} €</Descriptions.Item>
|
||||
</Descriptions>
|
||||
<b>{record.title}</b>
|
||||
<p>{record.description == null ? 'No description available' : record.description}</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<ListingsFilter
|
||||
onActivityFilter={setActivityFilter}
|
||||
onWatchListFilter={setWatchListFilter}
|
||||
onJobNameFilter={setJobNameFilter}
|
||||
onProviderFilter={setProviderFilter}
|
||||
/>
|
||||
<Input
|
||||
prefix={<IconSearch />}
|
||||
showClear
|
||||
className="listingsTable__search"
|
||||
placeholder="Search"
|
||||
onChange={handleFilterChange}
|
||||
/>
|
||||
<Table
|
||||
rowKey="id"
|
||||
empty={empty}
|
||||
hideExpandedColumn={false}
|
||||
sticky={{ top: 5 }}
|
||||
columns={columns}
|
||||
expandedRowRender={expandRowRender}
|
||||
dataSource={(tableData?.result || []).map((row) => {
|
||||
return {
|
||||
...row,
|
||||
reloadTable: loadTable,
|
||||
};
|
||||
})}
|
||||
onChange={(changeSet) => {
|
||||
if (changeSet?.extra?.changeType === 'sorter') {
|
||||
setSortData({
|
||||
field: changeSet.sorter.dataIndex,
|
||||
direction: changeSet.sorter.sortOrder === 'ascend' ? 'asc' : 'desc',
|
||||
});
|
||||
}
|
||||
}}
|
||||
pagination={{
|
||||
currentPage: page,
|
||||
//for now fixed
|
||||
pageSize,
|
||||
total: tableData?.totalNumber || 0,
|
||||
onPageChange: handlePageChange,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
14
ui/src/components/table/listings/ListingsTable.less
Normal file
@@ -0,0 +1,14 @@
|
||||
.listingsTable {
|
||||
&__search {
|
||||
margin-bottom: 1rem !important;
|
||||
}
|
||||
|
||||
&__expanded {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
&__toolbar {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
}
|
||||
34
ui/src/components/version/VersionBanner.jsx
Normal file
@@ -0,0 +1,34 @@
|
||||
import React from 'react';
|
||||
import { Collapse, Descriptions } from '@douyinfe/semi-ui';
|
||||
import { useSelector } from '../../services/state/store.js';
|
||||
import { MarkdownRender } from '@douyinfe/semi-ui';
|
||||
|
||||
import './VersionBanner.less';
|
||||
|
||||
export default function VersionBanner() {
|
||||
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
|
||||
return (
|
||||
<Collapse>
|
||||
<Collapse.Panel header="A new version of Fredy is available" itemKey="1" className="versionBanner">
|
||||
<div className="versionBanner__content">
|
||||
<p>A new version of Fredy is available. Update now to take advantage of the latest features and bug fixes.</p>
|
||||
<Descriptions row size="small">
|
||||
<Descriptions.Item itemKey="Your Version">{versionUpdate.localFredyVersion}</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Latest Version">{versionUpdate.version}</Descriptions.Item>
|
||||
<Descriptions.Item itemKey="Github Release">
|
||||
<a href={versionUpdate.url} target="_blank" rel="noreferrer">
|
||||
{versionUpdate.url}
|
||||
</a>{' '}
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
<p>
|
||||
<b>
|
||||
<small>Release Notes</small>
|
||||
</b>
|
||||
</p>
|
||||
<MarkdownRender raw={versionUpdate.body} />
|
||||
</div>
|
||||
</Collapse.Panel>
|
||||
</Collapse>
|
||||
);
|
||||
}
|
||||
7
ui/src/components/version/VersionBanner.less
Normal file
@@ -0,0 +1,7 @@
|
||||
.versionBanner {
|
||||
background: rgba(var(--semi-teal-1), 1);
|
||||
|
||||
&__content {
|
||||
overflow: auto;
|
||||
}
|
||||
}
|
||||
23
ui/src/hooks/screenWidth.js
Normal file
@@ -0,0 +1,23 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
export function useScreenWidth() {
|
||||
const [width, setWidth] = useState(window.innerWidth);
|
||||
|
||||
useEffect(() => {
|
||||
let timeoutId;
|
||||
|
||||
const handleResize = () => {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = setTimeout(() => setWidth(window.innerWidth), 100);
|
||||
};
|
||||
|
||||
window.addEventListener('resize', handleResize);
|
||||
|
||||
return () => {
|
||||
clearTimeout(timeoutId);
|
||||
window.removeEventListener('resize', handleResize);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return width;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const demoMode = {
|
||||
state: {
|
||||
demoMode: false,
|
||||
},
|
||||
reducers: {
|
||||
setDemoMode: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
demoMode: payload.demoMode,
|
||||
};
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getDemoMode() {
|
||||
try {
|
||||
const response = await xhrGet('/api/demo');
|
||||
this.setDemoMode(response.json);
|
||||
} catch (Exception) {
|
||||
console.error('Error while trying to get resource for api/demo. Error:', Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,25 +0,0 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const generalSettings = {
|
||||
state: {
|
||||
settings: {},
|
||||
},
|
||||
reducers: {
|
||||
//only admins
|
||||
setGeneralSettings: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
settings: payload,
|
||||
};
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getGeneralSettings() {
|
||||
try {
|
||||
const response = await xhrGet('/api/admin/generalSettings');
|
||||
this.setGeneralSettings(response.json);
|
||||
} catch (Exception) {
|
||||
console.error('Error while trying to get resource for api/admin/generalSettings. Error:', Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,57 +0,0 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const jobs = {
|
||||
state: {
|
||||
jobs: [],
|
||||
insights: {},
|
||||
processingTimes: {},
|
||||
},
|
||||
reducers: {
|
||||
setJobs: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
jobs: Object.freeze(payload),
|
||||
};
|
||||
},
|
||||
setProcessingTimes: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
processingTimes: Object.freeze(payload),
|
||||
};
|
||||
},
|
||||
setJobInsights: (state, payload, jobId) => {
|
||||
return {
|
||||
...state,
|
||||
insights: {
|
||||
...state.insights,
|
||||
[jobId]: Object.freeze(payload),
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getJobs() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs');
|
||||
this.setJobs(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
async getProcessingTimes() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs/processingTimes');
|
||||
this.setProcessingTimes(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/processingTimes. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
async getInsightDataForJob(jobId) {
|
||||
try {
|
||||
const response = await xhrGet(`/api/jobs/insights/${jobId}`);
|
||||
this.setJobInsights(response.json, jobId);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs/insights. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,19 +0,0 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const notificationAdapter = {
|
||||
state: [],
|
||||
reducers: {
|
||||
setAdapter: (state, payload) => {
|
||||
return [...Object.freeze(payload)];
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getAdapter() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs/notificationAdapter');
|
||||
this.setAdapter(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs/notificationAdapter. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,19 +0,0 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const provider = {
|
||||
state: [],
|
||||
reducers: {
|
||||
setProvider: (state, payload) => {
|
||||
return [...Object.freeze(payload)];
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getProvider() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs/provider');
|
||||
this.setProvider(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs/provider. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,40 +0,0 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const user = {
|
||||
state: {
|
||||
users: [],
|
||||
currentUser: null,
|
||||
},
|
||||
reducers: {
|
||||
//only admins
|
||||
setUsers: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
users: payload,
|
||||
};
|
||||
},
|
||||
setCurrentUser: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
currentUser: Object.freeze(payload),
|
||||
};
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getUsers() {
|
||||
try {
|
||||
const response = await xhrGet('/api/admin/users');
|
||||
this.setUsers(response.json);
|
||||
} catch (Exception) {
|
||||
console.error('Error while trying to get resource for api/admin/users. Error:', Exception);
|
||||
}
|
||||
},
|
||||
async getCurrentUser() {
|
||||
try {
|
||||
const response = await xhrGet('/api/login/user');
|
||||
this.setCurrentUser(response.json);
|
||||
} catch (Exception) {
|
||||
console.error('Error while trying to get resource for api/login/user. Error:', Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,29 +0,0 @@
|
||||
import { notificationAdapter } from './models/notificationAdapter';
|
||||
import { generalSettings } from './models/generalSettings';
|
||||
import createLoadingPlugin from '@rematch/loading';
|
||||
import { provider } from './models/provider';
|
||||
import { createLogger } from 'redux-logger';
|
||||
import { jobs } from './models/jobs';
|
||||
import { user } from './models/user';
|
||||
import { demoMode } from './models/demoMode.js';
|
||||
import { init } from '@rematch/core';
|
||||
const middleware = [];
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
middleware.push(createLogger({ duration: false, collapsed: (getState, action, logEntry) => !logEntry.error }));
|
||||
}
|
||||
const store = init({
|
||||
name: 'fredy',
|
||||
models: {
|
||||
notificationAdapter,
|
||||
generalSettings,
|
||||
demoMode,
|
||||
provider,
|
||||
jobs,
|
||||
user,
|
||||
},
|
||||
plugins: [createLoadingPlugin({})],
|
||||
redux: {
|
||||
middlewares: middleware,
|
||||
},
|
||||
});
|
||||
export const reduxStore = store;
|
||||