Compare commits

..

28 Commits

Author SHA1 Message Date
orangecoding
42cd4fa0ae next release version 2025-09-27 18:15:58 +02:00
orangecoding
6d96fd2bf8 Merge branch 'master' of github.com:orangecoding/fredy 2025-09-27 18:15:42 +02:00
orangecoding
ff1d2317a1 improve default puppeteer header 2025-09-27 18:15:28 +02:00
orangecoding
a47fa41278 fixing smaller problems in apprise and mattermost 2025-09-27 18:07:48 +02:00
orangecoding
9654e56846 improving some labels 2025-09-27 18:01:42 +02:00
Christian Kellner
43094640a8 Update README.md 2025-09-27 14:27:25 +02:00
orangecoding
fa234d2d78 fixing code style issues in new discord adapter 2025-09-27 14:24:05 +02:00
orangecoding
7cb0d6e382 next release version 2025-09-27 14:22:09 +02:00
mari
d79f8d2664 Add Discord webhook adapter (#196)
* Add Discord webhook adapter
2025-09-27 14:20:43 +02:00
Thomas Brockmöller
4d37e890ab Add provider for Regionalimmobilien24 (#197) 2025-09-27 14:19:37 +02:00
Thomas Brockmöller
7589f20a18 Add sparkasse immobilien (#199) 2025-09-27 09:43:24 +02:00
Thomas Brockmöller
702ffabc1a Fix and improve immowelt/immonet provider (#194)
* Fix and improve immowelt provider

* Add description to immonet provider

* Fix tests and improve readability
2025-09-27 09:42:08 +02:00
orangecoding
9387de1cd9 next version 2025-09-26 13:09:22 +02:00
orangecoding
facd683d45 santizing ntfy header 2025-09-26 13:07:54 +02:00
Christian Kellner
8324357edb Improvements (#193)
* improving release banner

* renaming general to settings

* fixing working hours if they go to next day

* fixing comparing versions

* upgrade dependencies
2025-09-26 10:45:55 +02:00
Christian Kellner
67af7c7dc5 next version 2025-09-25 15:06:38 +02:00
Christian Kellner
6f5b52f3ad Merge branch 'master' of github.com:orangecoding/fredy 2025-09-25 15:06:25 +02:00
Christian Kellner
89d239c360 New Listings view (#192)
* completing found listings

---------

Co-authored-by: Christian Kellner <Christian.Kellner1@ibm.com>
2025-09-25 15:03:47 +02:00
Thomas Brockmöller
dd5c5b29d9 Fix address value in similarity filtering (#191)
* Fix address field in similarity filter
2025-09-25 15:02:00 +02:00
Christian Kellner
0cb2f48645 Merge branch 'master' of github.com:orangecoding/fredy 2025-09-25 09:47:16 +02:00
orangecoding
3f294b8099 next release version 2025-09-22 20:56:42 +02:00
Christian Kellner
11fd18e76a Puppeteer improvements (#186)
* improving puppeteer handling

* upgrade dependencies

* reduce logging

* upgrade nanoid
2025-09-22 20:53:00 +02:00
Christian Kellner
c839f3abc9 Check if a listing is still active (#184)
* check if a listing is still active

* upgrade dependencies
2025-09-22 09:57:50 +02:00
orangecoding
28eddc5d7f next release version 2025-09-20 19:49:32 +02:00
Iaroslav Postovalov
0ca9c5ae02 Add health check for Docker container (#179)
- Introduced `HealthCheck` in `docker-compose.yml` to monitor container status.
- Added a test step to validate container's health using Docker Compose in the GitHub workflow.
- Updated `Dockerfile` to include `curl` for health check commands.
2025-09-20 19:39:48 +02:00
orangecoding
a7d0037edd next release version 2025-09-20 19:37:47 +02:00
orangecoding
f339a2e2cf adding version banner to check if a new version of fredy is available 2025-09-20 19:37:27 +02:00
Christian Kellner
0d2b21c789 improve security by shortn the cookie ttl 2025-09-04 12:52:18 +02:00
62 changed files with 1589 additions and 318 deletions

View File

@@ -57,3 +57,41 @@ jobs:
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
# Test container health with docker compose
- name: Test container with docker compose
run: |
echo "Starting container with docker compose..."
docker compose up --build -d
echo "Waiting for container to be ready (60 seconds for start_period)..."
sleep 60
echo "Monitoring container health for 30 seconds..."
SECONDS_ELAPSED=0
HEALTH_CHECK_INTERVAL=5
TOTAL_DURATION=30
while [ $SECONDS_ELAPSED -lt $TOTAL_DURATION ]; do
HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' fredy 2>/dev/null || echo "not_found")
CONTAINER_STATUS=$(docker inspect --format='{{.State.Status}}' fredy 2>/dev/null || echo "not_found")
echo "[$SECONDS_ELAPSED/$TOTAL_DURATION sec] Container: $CONTAINER_STATUS, Health: $HEALTH_STATUS"
# Check if container is not running or unhealthy
if [ "$CONTAINER_STATUS" != "running" ]; then
echo "Container stopped running! Status: $CONTAINER_STATUS"
docker compose logs fredy
exit 1
fi
if [ "$HEALTH_STATUS" = "unhealthy" ]; then
echo "Container is unhealthy!"
docker compose logs fredy
docker inspect --format='{{json .State.Health}}' fredy | jq
exit 1
fi
sleep $HEALTH_CHECK_INTERVAL
SECONDS_ELAPSED=$((SECONDS_ELAPSED + HEALTH_CHECK_INTERVAL))
done
docker compose down

View File

@@ -2,9 +2,10 @@ FROM node:22-slim
WORKDIR /fredy
# Install Chromium without extra recommended packages and clean apt cache
# Install Chromium and curl without extra recommended packages and clean apt cache
# curl is needed for the health check
RUN apt-get update \
&& apt-get install -y --no-install-recommends chromium \
&& apt-get install -y --no-install-recommends chromium curl \
&& rm -rf /var/lib/apt/lists/*
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \

View File

@@ -21,15 +21,13 @@ Finding an apartment or house in Germany can be stressful and
time-consuming.\
**Fredy** makes it easier: it automatically scrapes **ImmoScout24,
Immowelt, Immonet, eBay Kleinanzeigen, and WG-Gesucht** and notifies you
instantly via **Slack, Telegram, Email, ntfy, and more** when new
instantly via **Slack, Telegram, Email, ntfy, discord and more** when new
listings appear.
With a modern architecture, Fredy provides a **clean Web UI**, removes
duplicates across platforms, and stores results so you never see the
same listing twice.
------------------------------------------------------------------------
## ✨ Key Features
@@ -37,7 +35,7 @@ same listing twice.
- 🏠 Scrapes **ImmoScout24, Immowelt, Immonet, eBay Kleinanzeigen,
WG-Gesucht**
- ⚡ Instant notifications: Slack, Telegram, Email (SendGrid,
Mailjet), ntfy
Mailjet), ntfy, discord
- 🔎 Uses the **ImmoScout Mobile API** (reverse engineered)
- 🌍 Runs anywhere: Docker, Node.js, self-hosted
- 🖥️ Intuitive **Web UI** to manage searches
@@ -131,7 +129,7 @@ picks up the newest listings first.
### Adapter 📡
An **adapter** is the channel through which Fredy notifies you (Slack,
Telegram, Email, ntfy, ...).\
Telegram, Email, ntfy, discord ...).\
Each adapter has its own configuration (e.g. API keys, webhook URLs).\
You can use multiple adapters at once --- Fredy will send new listings
through all of them.

View File

@@ -1 +1 @@
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null,"sqlitepath":"/db"}
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}

View File

@@ -11,5 +11,12 @@ services:
- ./conf:/conf
- ./db:/db
ports:
- 9998:9998
- "9998:9998"
restart: unless-stopped
healthcheck:
# The container will immediately stop when health check fails after retries
test: ["CMD-SHELL", "curl --fail --silent --show-error --max-time 5 http://localhost:9998/ || exit 1"]
interval: 120s
timeout: 10s
retries: 1
start_period: 10s

View File

@@ -1,16 +1,20 @@
import fs from 'fs';
import path from 'path';
import { config } from './lib/utils.js';
import { config, getProviders, refreshConfig } from './lib/utils.js';
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import * as jobStorage from './lib/services/storage/jobStorage.js';
import FredyRuntime from './lib/FredyRuntime.js';
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
import logger from './lib/services/logger.js';
import { bus } from './lib/services/events/event-bus.js';
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
// Load configuration before any other startup steps
await refreshConfig();
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
const rawDir = config.sqlitepath || '/db';
@@ -23,8 +27,9 @@ if (!fs.existsSync(absDir)) {
// Run DB migrations once at startup and block until finished
await runMigrations();
const providersPath = './lib/provider';
const provider = fs.readdirSync(providersPath).filter((file) => file.endsWith('.js'));
// Load provider modules once at startup
const providers = await getProviders();
//assuming interval is always in minutes
const INTERVAL = config.interval * 60 * 1000;
@@ -38,13 +43,11 @@ if (config.demoMode) {
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
const fetchedProvider = await Promise.all(
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${providersPath}/${pro}`)),
);
ensureAdminUserExists();
ensureDemoUserExists();
await initTrackerCron();
//do not wait for this to finish, let it run in the background
initActiveCheckerCron();
bus.on('jobs:runAll', () => {
logger.debug('Running Fredy Job manually');
@@ -61,11 +64,17 @@ const execute = () => {
.filter((job) => job.enabled)
.forEach((job) => {
job.provider
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
.filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
.forEach(async (prov) => {
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
pro.init(prov, job.blacklist);
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyRuntime(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
});
});
} else {

View File

@@ -107,7 +107,7 @@ class FredyRuntime {
}
return !similar;
});
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, listings.address));
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
return filteredList;
}

View File

@@ -3,10 +3,11 @@ import { authInterceptor, cookieSession, adminInterceptor } from './security.js'
import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
import { analyticsRouter } from './routes/analyticsRouter.js';
import { providerRouter } from './routes/providerRouter.js';
import { versionRouter } from './routes/versionRouter.js';
import { loginRouter } from './routes/loginRoute.js';
import { config } from '../utils.js';
import { userRouter } from './routes/userRoute.js';
import { jobRouter } from './routes/jobRouter.js';
import { config } from '../utils.js';
import bodyParser from 'body-parser';
import restana from 'restana';
import files from 'serve-static';
@@ -14,6 +15,7 @@ import path from 'path';
import { getDirName } from '../utils.js';
import { demoRouter } from './routes/demoRouter.js';
import logger from '../services/logger.js';
import { listingsRouter } from './routes/listingsRouter.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../ui/public'));
const PORT = config.port || 9998;
@@ -23,6 +25,9 @@ service.use(cookieSession());
service.use(staticService);
service.use('/api/admin', authInterceptor());
service.use('/api/jobs', authInterceptor());
service.use('/api/version', authInterceptor());
service.use('/api/listings', authInterceptor());
// /admin can only be accessed when user is having admin permissions
service.use('/api/admin', adminInterceptor());
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
@@ -30,8 +35,10 @@ service.use('/api/admin/generalSettings', generalSettingsRouter);
service.use('/api/jobs/provider', providerRouter);
service.use('/api/jobs/insights', analyticsRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/version', versionRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
service.use('/api/listings', listingsRouter);
//this route is unsecured intentionally as it is being queried from the login page
service.use('/api/demo', demoRouter);

View File

@@ -0,0 +1,23 @@
import restana from 'restana';
import * as listingStorage from '../../services/storage/listingsStorage.js';
import { isAdmin as isAdminFn } from '../security.js';
const service = restana();
const listingsRouter = service.newRouter();
listingsRouter.get('/table', async (req, res) => {
const { page, pageSize = 50, filter, sortfield = null, sortdir = 'asc' } = req.query || {};
const result = listingStorage.queryListings({
page: page ? parseInt(page, 10) : 1,
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
filter: filter || undefined,
sortField: sortfield || null,
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
userId: req.session.currentUser,
isAdmin: isAdminFn(req),
});
res.body = result;
res.send();
});
export { listingsRouter };

View File

@@ -0,0 +1,31 @@
import restana from 'restana';
import fetch from 'node-fetch';
import { getPackageVersion } from '../../utils.js';
import semver from 'semver';
const service = restana();
const versionRouter = service.newRouter();
versionRouter.get('/', async (req, res) => {
const versionPayload = await getCurrentVersionFromGithub();
res.body = versionPayload == null ? { newVersion: false } : versionPayload;
res.send();
});
async function getCurrentVersionFromGithub() {
const raw = await fetch('https://api.github.com/repos/orangecoding/fredy/releases/latest');
const data = await raw.json();
const localFredyVersion = await getPackageVersion();
if (data.tag_name == null || semver.gte(localFredyVersion, data.tag_name)) {
return null;
}
return {
newVersion: true,
version: data.tag_name,
url: data.html_url,
body: data.body,
localFredyVersion,
};
}
export { versionRouter };

View File

@@ -37,7 +37,7 @@ const cookieSession$0 = (userId) => {
name: 'fredy-admin-session',
keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
userId,
maxAge: 8 * 60 * 60 * 1000, // 8 hours
maxAge: 2 * 60 * 60 * 1000, // 2 hours
});
};
export { cookieSession$0 as cookieSession };

View File

@@ -8,7 +8,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
const jobName = job == null ? jobKey : job.name;
const promises = newListings.map((newListing) => {
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
return fetch(server, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },

View File

@@ -0,0 +1,130 @@
import fetch from 'node-fetch';
import { getJob } from '../../services/storage/jobStorage.js';
import { markdown2Html } from '../../services/markdown.js';
import { normalizeImageUrl } from '../../utils.js';
/**
* Generates an idempotent decimal color code. The input string-based color code is
* generated using the djb2 hash algorithm.
*
* @param {string} str - Input string as color code base
* @returns {number} Generated decimal color code (0 - 16777215)
*/
const generateColorFromString = (str) => {
let hash = 5381; // initial value
const input = String(str);
for (let i = 0; i < input.length; i++) {
// hash * 33 + charCode
hash = (hash << 5) + hash + input.charCodeAt(i);
// Ensure the hash is 32 bit
hash |= 0;
}
let positiveHash = hash >>> 0;
const maxColorValue = 16777215;
const colorDecimal = positiveHash % maxColorValue;
return colorDecimal;
};
/**
* Creates an embed per listing
* (-> see https://birdie0.github.io/discord-webhooks-guide/structure/embeds.html).
*
* @param {string} jobKey - Key of job (used to set embed color)
* @param {object} listing - Object holding listing details
* @returns {object} Discord webhook embed
*/
const buildEmbed = (jobKey, listing) => {
const maxTitleLength = 252; // Max embed title length is 256 characters
let title = String(listing.title ?? 'N/A');
if (title.length > maxTitleLength) {
title = title.substring(0, maxTitleLength) + '...';
}
const fields = [
{
name: 'Price',
value: String(listing.price ?? 'n/a'),
inline: true,
},
{
name: 'Size',
value: listing?.size?.replace(/2m/g, 'm²') ?? 'n/a',
inline: true,
},
{
name: 'Address',
value: String(listing.address ?? 'n/a'),
inline: true,
},
];
const embed = {
title: title,
color: generateColorFromString(jobKey),
url: listing.link,
fields: fields,
};
if (listing.image) {
embed.image = {
url: normalizeImageUrl(listing.image),
};
}
return embed;
};
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const adapter = notificationConfig.find((adapter) => adapter.id === config.id);
const webhookUrl = adapter?.fields?.webhookUrl;
if (!webhookUrl || newListings.length === 0) return Promise.resolve([]);
const job = getJob(jobKey);
const jobName = job?.name || jobKey;
const embeds = newListings.map((listing) => buildEmbed(jobKey, listing));
const maxEmbedsPerMessage = 10; // Discord only allows up to 10 embeds
const webhookPromises = [];
for (let i = 0; i < embeds.length; i += maxEmbedsPerMessage) {
// Send multiple Discord messages with up to 10 embeds per message
const embedChunk = embeds.slice(i, i + maxEmbedsPerMessage);
const content = i === 0 ? `*${jobName}:* ${serviceName} found **${newListings.length}** new listings.` : '';
const body = JSON.stringify({
content: content,
embeds: embedChunk,
});
const fetchPromise = fetch(webhookUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body,
}).catch((error) => {
console.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
return Promise.reject(new Error(`Webhook failed: ${error.message}`));
});
webhookPromises.push(fetchPromise);
}
return Promise.allSettled(webhookPromises);
};
export const config = {
id: 'discord_webhook',
name: 'Discord Webhook',
readme: markdown2Html('lib/notification/adapter/discord_webhook.md'),
description: 'Fredy will send new listings to the Discord channel of your choice.',
fields: {
webhookUrl: {
type: 'text',
label: 'Webhook URL',
description: 'The URL of the Discord webhook to send messages to.',
},
},
};

View File

@@ -0,0 +1,4 @@
### Discord Adapter
To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
Once you have created a webhook, copy and paste the webhook URL.

View File

@@ -13,10 +13,10 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
return fetch(webhook, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: {
body: JSON.stringify({
channel: channel,
text: message,
},
}),
});
};
export const config = {

View File

@@ -15,11 +15,17 @@ Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$'
Price: ${newListing.price}
Link: ${newListing.link}`;
const sanitizeHeaderValue = (value) =>
String(value ?? '')
.replace(/[\r\n]+/g, ' ')
.replace(/[^\x20-\x7E]/g, ' ')
.trim();
const headers = {
Title: newListing.title,
Priority: String(priority),
Tags: `${serviceName},${jobName}`,
Click: newListing.link,
Title: sanitizeHeaderValue(newListing.title),
Priority: sanitizeHeaderValue(priority),
Tags: sanitizeHeaderValue(`${serviceName},${jobName}`),
Click: sanitizeHeaderValue(newListing.link),
};
if (newListing.image && typeof newListing.image === 'string') {

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
@@ -29,8 +30,8 @@ function normalizePrice(price) {
return result[0];
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -49,6 +50,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -24,8 +25,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -46,6 +47,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
@@ -11,8 +12,8 @@ function normalize(o) {
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
@@ -28,9 +29,11 @@ const config = {
address: 'div[data-testid="cardmfe-description-box-address"] | trim',
image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
link: 'button@data-base',
description: 'div[data-testid="cardmfe-description-text-test-id"] | trim',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -35,8 +35,11 @@
*
*/
import utils, { buildHash } from '../utils.js';
import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
import { buildHash, isOneOf } from '../utils.js';
import {
convertImmoscoutListingToMobileListing,
convertWebToMobile,
} from '../services/immoscout/immoscout-web-translator.js';
import logger from '../services/logger.js';
let appliedBlackList = [];
@@ -77,6 +80,25 @@ async function getListings(url) {
});
}
async function isListingActive(link) {
const result = await fetch(convertImmoscoutListingToMobileListing(link), {
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
},
});
if (result.status === 200) {
return 1;
}
if (result.status === 404) {
return 0;
}
logger.warn('Unknown status for immoscout listing', link);
return -1;
}
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
@@ -87,7 +109,7 @@ function normalize(o) {
return Object.assign(o, { id, title, address });
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
return !isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
@@ -104,6 +126,7 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
getListings: getListings,
activeTester: isListingActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -14,8 +15,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -35,6 +36,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -8,8 +9,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
@@ -25,11 +26,13 @@ const config = {
size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
link: 'a@href',
description: 'div[data-testid="cardmfe-description-text-test-id"] > div:nth-of-type(2) | removeNewline | trim',
address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
image: 'div[data-testid="cardMfe-card-pictureBox-opacity"] img@src',
image: 'div[data-testid="cardmfe-picture-box-opacity-layer-test-id"] img@src',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
let appliedBlacklistedDistricts = [];
@@ -11,10 +12,10 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
const isBlacklistedDistrict =
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
appliedBlacklistedDistricts.length === 0 ? false : isOneOf(o.description, appliedBlacklistedDistricts);
return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
@@ -36,6 +37,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const metaInformation = {
name: 'Ebay Kleinanzeigen',

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -15,14 +16,14 @@ function normalize(o) {
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
return !isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
crawlContainer: '.col-12.mb-4',
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
waitForSelector: '.nbk-section',
waitForSelector: 'div[data-live-name-value="SearchList"]',
crawlFields: {
id: 'a@href',
title: 'a@title | removeNewline | trim',
@@ -33,6 +34,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -0,0 +1,49 @@
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
const id = buildHash(o.id, o.price);
const address = o.address?.replace(/^adresse /i, '') ?? null;
const title = o.title || 'No title available';
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
var urlReg = new RegExp(/url\((.*?)\)/gim);
const image = o.image != null ? urlReg.exec(o.image)[1] : null;
return Object.assign(o, { id, address, title, link, image });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.listentry-content',
sortByDateParam: null, // sort by date is standard
waitForSelector: 'body',
crawlFields: {
id: '.listentry-iconbar-share@data-sid | trim',
title: 'h2 | trim',
price: '.listentry-details-price .listentry-details-v | trim',
size: '.listentry-details-size .listentry-details-v | trim',
address: '.listentry-adress | trim',
image: '.listentry-img@style',
link: '.shariff@data-url',
description: '.listentry-extras | trim',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
export const metaInformation = {
name: 'Regionalimmobilien24',
baseUrl: 'https://www.regionalimmobilien24.de/',
id: 'regionalimmobilien24',
};
export { config };

46
lib/provider/sparkasse.js Executable file
View File

@@ -0,0 +1,46 @@
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
const originalId = o.id.split('/').pop().replace('.html', '');
const id = buildHash(originalId, o.price);
const size = o.size?.replace(' Wohnfläche', '') ?? null;
const title = o.title || 'No title available';
const link = o.link != null ? `https://immobilien.sparkasse.de${o.link}` : config.url;
return Object.assign(o, { id, size, title, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.estate-list-item-row',
sortByDateParam: 'sortBy=date_desc',
waitForSelector: 'body',
crawlFields: {
id: 'div[data-testid="estate-link"] a@href',
title: 'h3 | trim',
price: '.estate-list-price | trim',
size: '.estate-mainfact:first-child span | trim',
address: 'h6 | trim',
image: '.estate-list-item-image-container img@src',
link: 'div[data-testid="estate-link"] a@href',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
export const metaInformation = {
name: 'Sparkasse Immobilien',
baseUrl: 'https://immobilien.sparkasse.de/',
id: 'sparkasse',
};
export { config };

View File

@@ -1,4 +1,5 @@
import utils, { buildHash } from '../utils.js';
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -10,8 +11,8 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
}
@@ -31,6 +32,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,7 +1,7 @@
import { removeJobsByUserId } from './storage/jobStorage.js';
import { config } from '../utils.js';
import { getUsers } from './storage/userStorage.js';
import logger from './logger.js';
import { removeJobsByUserId } from '../storage/jobStorage.js';
import { config } from '../../utils.js';
import { getUsers } from '../storage/userStorage.js';
import logger from '../logger.js';
import cron from 'node-cron';
/**

View File

@@ -0,0 +1,13 @@
import cron from 'node-cron';
import runActiveChecker from '../listings/listingActiveService.js';
async function runTask() {
await runActiveChecker();
}
export async function initActiveCheckerCron() {
//run directly on start
await runTask();
// then every day at 1 am
cron.schedule('0 1 * * *', runTask);
}

View File

@@ -1,6 +1,6 @@
import cron from 'node-cron';
import { config, inDevMode } from '../../utils.js';
import { trackMainEvent } from './Tracker.js';
import { trackMainEvent } from '../tracking/Tracker.js';
async function runTask() {
//make sure to only send tracking events if the user gave us the green light and we are not in dev mode

View File

@@ -9,12 +9,12 @@ export function loadParser(text) {
export function parse(crawlContainer, crawlFields, text, url) {
if (!text) {
logger.warn('No content found for ', url);
logger.debug('No content found for ', url);
return null;
}
if (!crawlContainer || !crawlFields) {
logger.warn('Cannot parse, selector was empty for url ', url);
logger.debug('Cannot parse, selector was empty for url ', url);
return null;
}

View File

@@ -2,30 +2,56 @@ import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
import logger from '../logger.js';
import fs from 'fs';
import os from 'os';
import path from 'path';
puppeteer.use(StealthPlugin());
export default async function execute(url, waitForSelector, options) {
let browser;
let page;
let result = null;
let userDataDir;
let removeUserDataDir = false;
try {
debug(`Sending request to ${url} using Puppeteer.`);
// Prepare a dedicated temporary userDataDir to avoid leaking /tmp/.org.chromium.* dirs
if (options && options.userDataDir) {
userDataDir = options.userDataDir;
removeUserDataDir = !!options.cleanupUserDataDir;
} else {
const prefix = path.join(os.tmpdir(), 'puppeteer-fredy-');
userDataDir = fs.mkdtempSync(prefix);
removeUserDataDir = true;
}
browser = await puppeteer.launch({
headless: options.puppeteerHeadless ?? true,
args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox'],
args: [
'--no-sandbox',
'--disable-gpu',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-crash-reporter',
],
timeout: options.puppeteerTimeout || 30_000,
userDataDir,
});
let page = await browser.newPage();
page = await browser.newPage();
await page.setExtraHTTPHeaders(DEFAULT_HEADER);
const response = await page.goto(url, {
waitUntil: 'domcontentloaded',
});
let pageSource;
//if we're extracting data from a spa, we must wait for the selector
// if we're extracting data from a SPA, we must wait for the selector
if (waitForSelector != null) {
await page.waitForSelector(waitForSelector);
const selectorTimeout = options?.puppeteerSelectorTimeout ?? options?.puppeteerTimeout ?? 30_000;
await page.waitForSelector(waitForSelector, { timeout: selectorTimeout });
pageSource = await page.evaluate((selector) => {
return document.querySelector(selector).innerHTML;
const el = document.querySelector(selector);
return el ? el.innerHTML : '';
}, waitForSelector);
} else {
pageSource = await page.content();
@@ -35,16 +61,35 @@ export default async function execute(url, waitForSelector, options) {
if (botDetected(pageSource, statusCode)) {
logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
return null;
result = null;
} else {
result = pageSource || (await page.content());
}
return await page.content();
} catch (error) {
logger.error('Error executing with puppeteer executor', error);
return null;
result = null;
} finally {
if (browser != null) {
await browser.close();
try {
if (page) {
await page.close();
}
} catch {
// ignore
}
try {
if (browser != null) {
await browser.close();
}
} catch {
// ignore
}
try {
if (removeUserDataDir && userDataDir) {
await fs.promises.rm(userDataDir, { recursive: true, force: true });
}
} catch {
// ignore
}
}
return result;
}

View File

@@ -3,12 +3,25 @@ import logger from '../logger.js';
let debuggingOn = false;
export const DEFAULT_HEADER = {
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' +
'AppleWebKit/537.36 (KHTML, like Gecko) ' +
'Chrome/140.0.7339.207 Safari/537.36',
'Sec-CH-UA': '"Chromium";v="140", "Not.A/Brand";v="8"',
'Sec-CH-UA-Mobile': '?0',
'Sec-CH-UA-Platform': '"Windows"',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
Referer: 'https://www.google.com/',
DNT: '1',
TE: 'trailers',
};
export const setDebug = (options) => {

View File

@@ -60,6 +60,7 @@ https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T
https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC?
*/
import queryString from 'query-string';
import { nullOrEmpty } from '../../utils.js';
const PARAM_NAME_MAP = {
heatingtypes: 'heatingtypes',
@@ -193,3 +194,14 @@ export function convertWebToMobile(webUrl) {
return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
}
export function convertImmoscoutListingToMobileListing(url) {
if (nullOrEmpty(url)) {
return null;
}
return url.replace(
/^https:\/\/www\.immobilienscout24\.de\/expose\//,
'https://api.mobile.immobilienscout24.de/expose/',
);
}

View File

@@ -0,0 +1,104 @@
import { deactivateListings, getActiveOrUnknownListings } from '../storage/listingsStorage.js';
import { getProviders } from '../../utils.js';
import logger from '../../services/logger.js';
/**
* Runs the active-listing checker:
* 1) Loads all listings with unknown or active status.
* 2) Resolves each listing's provider and calls its `activeTester(link)`.
* 3) Collects listings that are no longer active and deactivates them in one batch.
*
* Concurrency: network-bound checks are executed with a configurable concurrency limit.
*
* @param {object} [opts]
* @param {number} [opts.concurrency=8] Max number of parallel activeTester calls.
* @returns {Promise<void>}
*/
export default async function runActiveChecker(opts = {}) {
const { concurrency = 4 } = opts;
const listings = getActiveOrUnknownListings();
if (!Array.isArray(listings) || listings.length === 0) {
logger.debug('No listings to check.');
return;
}
const providers = await getProviders();
if (!Array.isArray(providers) || providers.length === 0) {
logger.warn('No providers available. Skipping active checks.');
return;
}
// Build a map for O(1) provider lookup by id
/** @type {Record<string, any>} */
const providerById = Object.create(null);
for (const p of providers) {
const id = p?.metaInformation?.id;
if (id) providerById[id] = p;
}
// Small generic mapLimit to cap concurrency without extra deps
/**
* @template T, R
* @param {T[]} items
* @param {number} limit
* @param {(item: T, index: number) => Promise<R>} worker
* @returns {Promise<R[]>}
*/
async function mapLimit(items, limit, worker) {
const results = new Array(items.length);
let next = 0;
async function runOne() {
while (next < items.length) {
const i = next++;
try {
results[i] = await worker(items[i], i);
} catch (err) {
results[i] = /** @type {any} */ (err);
}
}
}
const runners = Array.from({ length: Math.min(limit, items.length) }, runOne);
await Promise.all(runners);
return results;
}
/** @type {string[]} */
const listingsSetToInactive = [];
await mapLimit(listings, concurrency, async (listing) => {
const { provider: listingProviderId, link, id } = listing || {};
const matchedProvider = providerById[listingProviderId];
if (!matchedProvider) {
logger.warn('Could not find matching provider for', listingProviderId);
return;
}
const tester = matchedProvider?.config?.activeTester;
if (typeof tester !== 'function') {
logger.warn('No activeTester configured for', listingProviderId);
return;
}
// Contract: activeTester(link) returns 1 if active, 0 if inactive
let result;
try {
result = await tester(link);
} catch {
result = -1;
}
if (result === 0 && id) {
listingsSetToInactive.push(id);
}
});
if (listingsSetToInactive.length > 0) {
logger.info(`Setting ${listingsSetToInactive.length} listings to inactive.`);
deactivateListings(listingsSetToInactive);
} else {
logger.debug('No listings need to be set inactive.');
}
}

View File

@@ -0,0 +1,68 @@
import fetch from 'node-fetch';
import { randomBetween, sleep } from '../../utils.js';
const maxAttempts = 3;
/**
* Check if a listing is still active with up to 3 attempts and exponential backoff.
* Backoff waits are capped and the last wait is at most 2000 ms.
*
* Rules:
* - HTTP 200 => return 1
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
* - HTTP 404 => return 0
* - Other statuses or network errors => retry until attempts are exhausted
*
* @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
*/
export default async function checkIfListingIsActive(link) {
await sleep(randomBetween(50, 100));
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const res = await fetch(link, {
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
},
});
if (res.status === 200) {
return 1;
}
if (res.status === 401) return -1;
if (res.status === 403) return -1;
if (res.status === 404) return 0;
// For any other status, only retry if attempts remain
if (attempt < maxAttempts) {
await sleep(backoffDelay(attempt));
continue;
}
return 0;
} catch {
// Network error: retry if attempts remain
if (attempt < maxAttempts) {
await sleep(backoffDelay(attempt));
continue;
}
return 0;
}
}
return 0;
}
/**
* Exponential backoff delay with cap.
* attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
* @param {number} attempt 1-based attempt index
* @returns {number} delay in ms
*/
function backoffDelay(attempt) {
const base = 500;
const cap = 2000;
return Math.min(base * 2 ** (attempt - 1), cap);
}

View File

@@ -53,6 +53,36 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => {
).map((r) => r.hash);
};
/**
* Return a list of listing that either are active or have an unknown status
* to constantly check if they are still online
*
* @returns {string[]} Array of listings
*/
export const getActiveOrUnknownListings = () => {
return SqliteConnection.query(
`SELECT *
FROM listings
WHERE is_active is null OR is_active = 1 ORDER BY provider`,
);
};
/**
* Deactivates listings by setting is_active = 0 for all matching IDs.
*
* @param {string[]} ids - Array of listing IDs to deactivate.
* @returns {object[]} Result of the SQLite query execution.
*/
export const deactivateListings = (ids) => {
const placeholders = ids.map(() => '?').join(',');
return SqliteConnection.execute(
`UPDATE listings
SET is_active = 0
WHERE id IN (${placeholders})`,
ids,
);
};
/**
* Persist a batch of scraped listings for a given job and provider.
*
@@ -86,9 +116,9 @@ export const storeListings = (jobId, providerId, listings) => {
SqliteConnection.withTransaction((db) => {
const stmt = db.prepare(
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
link, created_at)
link, created_at, is_active)
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
@created_at)
@created_at, 1)
ON CONFLICT(job_id, hash) DO NOTHING`,
);
@@ -136,3 +166,88 @@ export const storeListings = (jobId, providerId, listings) => {
return str.replace(/\s*\([^)]*\)/g, '');
}
};
/**
* Query listings with pagination, filtering and sorting.
*
* @param {Object} params
* @param {number} [params.pageSize=50]
* @param {number} [params.page=1]
* @param {string} [params.filter]
* @param {string|null} [params.sortField=null] - One of: 'created_at','price','size','provider','title'.
* @param {('asc'|'desc')} [params.sortDir='asc']
* @param {string} [params.userId] - Current user id used to scope listings (ignored for admins).
* @param {boolean} [params.isAdmin=false] - When true, returns all listings.
* @returns {{ totalNumber:number, page:number, result:Object[] }}
*/
export const queryListings = ({
pageSize = 50,
page = 1,
filter,
sortField = null,
sortDir = 'asc',
userId = null,
isAdmin = false,
} = {}) => {
// sanitize inputs
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
const offset = (safePage - 1) * safePageSize;
// build WHERE filter across common text columns
const whereParts = [];
const params = { limit: safePageSize, offset };
// user scoping (non-admin only): restrict to listings whose job belongs to user
if (!isAdmin) {
params.userId = userId || '__NO_USER__';
whereParts.push(`(j.user_id = @userId)`);
}
if (filter && String(filter).trim().length > 0) {
params.filter = `%${String(filter).trim()}%`;
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
}
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
const whereSqlWithAlias = whereSql
.replace(/\btitle\b/g, 'l.title')
.replace(/\bdescription\b/g, 'l.description')
.replace(/\baddress\b/g, 'l.address')
.replace(/\bprovider\b/g, 'l.provider')
.replace(/\blink\b/g, 'l.link')
.replace(/\bj\.user_id\b/g, 'j.user_id');
// whitelist sortable fields to avoid SQL injection
const sortable = new Set(['created_at', 'price', 'size', 'provider', 'title', 'job_name', 'is_active']);
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
const orderSql = safeSortField ? `ORDER BY ${safeSortField} ${safeSortDir}` : 'ORDER BY created_at DESC';
const orderSqlWithAlias = orderSql
.replace(/\bcreated_at\b/g, 'l.created_at')
.replace(/\bprice\b/g, 'l.price')
.replace(/\bsize\b/g, 'l.size')
.replace(/\bprovider\b/g, 'l.provider')
.replace(/\btitle\b/g, 'l.title')
.replace(/\bjob_name\b/g, 'j.name');
// count total with same WHERE
const countRow = SqliteConnection.query(
`SELECT COUNT(1) as cnt
FROM listings l
LEFT JOIN jobs j ON j.id = l.job_id
${whereSqlWithAlias}`,
params,
);
const totalNumber = countRow?.[0]?.cnt ?? 0;
// fetch page
const rows = SqliteConnection.query(
`SELECT l.*, j.name AS job_name
FROM listings l
LEFT JOIN jobs j ON j.id = l.job_id
${whereSqlWithAlias}
${orderSqlWithAlias}
LIMIT @limit OFFSET @offset`,
params,
);
return { totalNumber, page: safePage, result: rows };
};

View File

@@ -0,0 +1,8 @@
// Migration: there needs to be a unique index on job_id and hash as only
// this makes the listing indeed unique
export function up(db) {
db.exec(`
ALTER TABLE listings ADD COLUMN is_active INTEGER DEFAULT 1;
`);
}

View File

@@ -1,9 +1,7 @@
import { getJobs } from '../storage/jobStorage.js';
import { getUniqueId } from './uniqueId.js';
import { config, inDevMode } from '../../utils.js';
import { config, getPackageVersion, inDevMode } from '../../utils.js';
import os from 'os';
import { readFileSync } from 'fs';
import { packageUp } from 'package-up';
import fetch from 'node-fetch';
import logger from '../logger.js';
@@ -77,15 +75,3 @@ function enrichTrackingObject(trackingObject) {
version,
};
}
async function getPackageVersion() {
try {
const packagePath = await packageUp();
const packageJson = readFileSync(packagePath, 'utf8');
const json = JSON.parse(packageJson);
return json.version;
} catch (error) {
logger.error('Error reading version from package.json', error);
}
return 'N/A';
}

View File

@@ -1,15 +1,37 @@
import { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { readFile } from 'fs/promises';
import { createHash } from 'crypto';
import { DEFAULT_CONFIG } from './defaultConfig.js';
import fs from 'fs';
import fs, { readFileSync } from 'fs';
import logger from './services/logger.js';
import { packageUp } from 'package-up';
const RE_GT = />/g;
const RE_WEBP = /\/format\/webp/gi;
const RE_EXT = /\.(jpe?g|png|gif)(\?.*)?$/i;
const HTTPS_PREFIX = 'https://';
const providersDirectoryPath = `${getDirName()}/provider`;
/**
* Lazily load all provider modules from the provider directory.
* Caches the resolved array to avoid re-importing on subsequent calls.
*
* @returns {Promise<any[]>} A list of loaded provider modules.
*/
let cachedProvidersPromise = null;
export function getProviders() {
if (!cachedProvidersPromise) {
/** @type {string[]} */
const providerFileNames = fs.readdirSync(providersDirectoryPath).filter((fileName) => fileName.endsWith('.js'));
cachedProvidersPromise = Promise.all(
providerFileNames.map((fileName) => import(pathToFileURL(path.join(providersDirectoryPath, fileName)).href)),
);
}
return cachedProvidersPromise;
}
/**
* Safely stringify a value to JSON for storage.
@@ -20,7 +42,7 @@ const HTTPS_PREFIX = 'https://';
* @param {T} v - Any JSON-serializable value.
* @returns {string|null} JSON string or null.
*/
export const toJson = (v) => (v == null ? null : JSON.stringify(v));
const toJson = (v) => (v == null ? null : JSON.stringify(v));
/**
* Safely parse JSON text coming from storage.
@@ -32,7 +54,7 @@ export const toJson = (v) => (v == null ? null : JSON.stringify(v));
* @param {T} fallback - Value to return when txt is null/invalid.
* @returns {T} Parsed value or fallback.
*/
export const fromJson = (txt, fallback) => {
const fromJson = (txt, fallback) => {
if (txt == null) return fallback;
try {
return JSON.parse(txt);
@@ -87,11 +109,22 @@ function timeStringToMs(timeString, now) {
}
/**
* Check whether current time is within configured working hours, or no hours are set.
* If working hours are missing or incomplete, returns true.
* @param {{workingHours?: {from?: string, to?: string}}} config
* @param {number} now - Epoch ms
* @returns {boolean}
* Determine whether the given timestamp is within the configured working hours, or return true when the window is not set.
* - If workingHours is missing or either 'from' or 'to' is empty/null, returns true.
* - Supports windows that cross midnight (e.g., from '23:00' to '06:00').
*
* Time parsing is based on the local timezone of the running process.
*
* @param {{workingHours?: {from?: string|null, to?: string|null}}} config - Configuration object containing working hours in 'HH:mm' format.
* @param {number} now - Epoch milliseconds to evaluate.
* @returns {boolean} True when execution is allowed at 'now'.
* @example
* // Same-day window
* duringWorkingHoursOrNotSet({ workingHours: { from: '08:00', to: '17:00' } }, someTime);
* @example
* // Window crossing midnight
* // For { from: '05:00', to: '00:30' } → 23:00 => true, 01:00 => false, 06:00 => true
* duringWorkingHoursOrNotSet({ workingHours: { from: '05:00', to: '00:30' } }, Date.now());
*/
function duringWorkingHoursOrNotSet(config, now) {
const { workingHours } = config;
@@ -100,7 +133,20 @@ function duringWorkingHoursOrNotSet(config, now) {
}
const toDate = timeStringToMs(workingHours.to, now);
const fromDate = timeStringToMs(workingHours.from, now);
return fromDate <= now && toDate >= now;
// If parsing fails (e.g., malformed time), be lenient and allow.
if (isNaN(toDate) || isNaN(fromDate)) {
return true;
}
if (toDate >= fromDate) {
// Same-day window (e.g., 08:00 - 17:00)
return now >= fromDate && now <= toDate;
}
// Window crosses midnight (e.g., 05:00 -> 00:30 next day)
// Accept if we are after 'from' today OR before 'to' today (which represents next day's cutoff).
return now >= fromDate || now <= toDate;
}
/**
@@ -196,22 +242,56 @@ const normalizeImageUrl = (url) => {
return u;
};
/**
* returns Fredy's version
* @returns {Promise<*|string>}
*/
async function getPackageVersion() {
try {
const packagePath = await packageUp();
const packageJson = readFileSync(packagePath, 'utf8');
const json = JSON.parse(packageJson);
return json.version;
} catch (error) {
logger.error('Error reading version from package.json', error);
}
return 'N/A';
}
/**
* Sleep helper
* @param {number} ms milliseconds to wait
* @returns {Promise<void>}
*/
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Return a random integer between min and max (inclusive).
* @param {number} min - Minimum integer value.
* @param {number} max - Maximum integer value.
* @returns {number} A random integer N where min <= N <= max.
*/
function randomBetween(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
// Call refreshConfig() from the application entrypoint during startup to populate config.
await refreshConfig();
export { isOneOf };
export { normalizeImageUrl };
export { inDevMode };
export { nullOrEmpty };
export { duringWorkingHoursOrNotSet };
export { getDirName };
export { config };
export { buildHash };
export default {
export {
isOneOf,
normalizeImageUrl,
inDevMode,
nullOrEmpty,
duringWorkingHoursOrNotSet,
getDirName,
sleep,
randomBetween,
config,
buildHash,
getPackageVersion,
toJson,
fromJson,
};

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "12.1.2",
"version": "12.3.1",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",
@@ -58,36 +58,37 @@
"dependencies": {
"@douyinfe/semi-icons": "^2.86.0",
"@douyinfe/semi-ui": "2.86.0",
"@sendgrid/mail": "8.1.5",
"@visactor/react-vchart": "^2.0.4",
"@visactor/vchart": "^2.0.4",
"@sendgrid/mail": "8.1.6",
"@visactor/react-vchart": "^2.0.5",
"@visactor/vchart": "^2.0.5",
"@visactor/vchart-semi-theme": "^1.12.2",
"@vitejs/plugin-react": "5.0.3",
"better-sqlite3": "^12.2.0",
"better-sqlite3": "^12.4.1",
"body-parser": "2.2.0",
"cheerio": "^1.1.2",
"cookie-session": "2.1.1",
"handlebars": "4.7.8",
"lodash": "4.17.21",
"markdown": "^0.5.0",
"nanoid": "5.1.5",
"nanoid": "5.1.6",
"node-cron": "^4.2.1",
"node-fetch": "3.3.2",
"node-mailjet": "6.0.9",
"p-throttle": "^8.0.0",
"package-up": "^5.0.0",
"puppeteer": "^24.22.0",
"puppeteer": "^24.22.3",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.3.1",
"react": "18.3.1",
"react-dom": "18.3.1",
"react-router": "7.9.1",
"react-router-dom": "7.9.1",
"react-router": "7.9.2",
"react-router-dom": "7.9.2",
"restana": "5.1.0",
"semver": "^7.7.2",
"serve-static": "2.2.0",
"slack": "11.0.2",
"vite": "7.1.6",
"vite": "7.1.7",
"x-var": "^3.0.1",
"zustand": "^5.0.8"
},
@@ -97,14 +98,14 @@
"@babel/preset-env": "7.28.3",
"@babel/preset-react": "7.27.1",
"chai": "6.0.1",
"eslint": "9.35.0",
"eslint": "9.36.0",
"eslint-config-prettier": "10.1.8",
"eslint-plugin-react": "7.37.5",
"esmock": "2.7.3",
"history": "5.3.0",
"husky": "9.1.7",
"less": "4.4.1",
"lint-staged": "16.1.6",
"lint-staged": "16.2.1",
"mocha": "11.7.2",
"nodemon": "^3.1.10",
"prettier": "3.6.2"

View File

@@ -0,0 +1,53 @@
import { expect } from 'chai';
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { mockFredy } from '../utils.js';
describe('FredyRuntime', () => {
afterEach(() => {
similarityCache.invalidateAllForTest();
});
after(() => {
similarityCache.stopCacheCleanup();
});
describe('_filterBySimilarListings', () => {
let fredyRuntime;
beforeEach(async () => {
const FredyRuntime = await mockFredy();
fredyRuntime = new FredyRuntime({}, null, 'dummy-provider', 'dummy-job', similarityCache);
});
it('should filter out listings with similar title and address already in cache', () => {
similarityCache.addCacheEntry('Penthouse', 'Mustermann Straße 1');
const listings = [
{ id: '1', title: 'Penthouse', address: 'Mustermann Straße 1' },
{ id: '2', title: 'Nice apartment', address: 'Mustermann Straße 15' },
];
const result = fredyRuntime._filterBySimilarListings(listings);
expect(result).to.have.length(1);
expect(result[0].id).to.equal('2');
expect(result[0].title).to.equal('Nice apartment');
expect(similarityCache.hasSimilarEntries('Nice apartment', 'Mustermann Straße 15')).to.be.true;
});
it('should handle listings with null or undefined address', () => {
const listings = [
{ id: '1', title: 'Penthouse', address: null },
{ id: '2', title: 'Nice apartment', address: undefined },
];
const result = fredyRuntime._filterBySimilarListings(listings);
expect(result).to.have.length(2);
expect(similarityCache.hasSimilarEntries('Penthouse', null)).to.be.true;
expect(similarityCache.hasSimilarEntries('Nice apartment', undefined)).to.be.true;
});
});
});

View File

@@ -8,31 +8,30 @@ describe('#immonet testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immonet, [], []);
it('should test immonet provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
provider.init(providerConfig.immonet, [], []);
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.address).to.be.not.empty;
});
resolve();
});
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.address).to.be.not.empty;
});
});
});

View File

@@ -8,33 +8,32 @@ describe('#immowelt testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test immowelt provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.immowelt, [], []);
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immowelt');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
expect(notify.size).that.does.include('m²');
}
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immowelt.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immowelt');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
expect(notify.size).that.does.include('m²');
}
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immowelt.de');
expect(notify.address).to.be.not.empty;
});
});
});

View File

@@ -0,0 +1,43 @@
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import { expect } from 'chai';
import * as provider from '../../lib/provider/regionalimmobilien24.js';
describe('#regionalimmobilien24 testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test regionalimmobilien24 provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.regionalimmobilien24, []);
const fredy = new Fredy(
provider.config,
null,
provider.metaInformation.id,
'regionalimmobilien24',
similarityCache,
);
const listing = await fredy.execute();
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('regionalimmobilien24');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.address).to.be.not.empty;
});
});
});

View File

@@ -0,0 +1,37 @@
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import { expect } from 'chai';
import * as provider from '../../lib/provider/sparkasse.js';
describe('#sparkasse testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test sparkasse provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.sparkasse, []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'sparkasse', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('sparkasse');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.address).to.be.not.empty;
});
});
});

View File

@@ -32,6 +32,14 @@
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
"enabled": true
},
"regionalimmobilien24": {
"url": "https://www.regionalimmobilien24.de/rostock/rostock/kaufen/haus/-/-/-/?rd=5",
"enabled": true
},
"sparkasse": {
"url": "https://immobilien.sparkasse.de/immobilien/treffer?marketingType=buy&objectType=flat&perimeter=10&usageType=residential&zipCityEstateId=62782__Hamburg",
"enabled": true
},
"wgGesucht": {
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
"enabled": true

View File

@@ -1,4 +1,4 @@
import utils from '../../lib/utils.js';
import { isOneOf, duringWorkingHoursOrNotSet } from '../../lib/utils.js';
import assert from 'assert';
import { expect } from 'chai';
@@ -8,30 +8,45 @@ const fakeWorkingHoursConfig = (from, to) => ({
from,
},
});
describe('utils', () => {
describe('#isOneOf()', () => {
it('should be false', () => {
assert.equal(utils.isOneOf('bla', ['blub']), false);
assert.equal(isOneOf('bla', ['blub']), false);
});
it('should be true', () => {
assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
assert.equal(isOneOf('bla blub blubber', ['bla']), true);
});
});
describe('#duringWorkingHoursOrNotSet()', () => {
it('should be false', () => {
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
});
it('should be true', () => {
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
});
it('should be true if nothing set', () => {
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
});
it('should be true if only to is set', () => {
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
});
it('should be true if only from is set', () => {
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
});
it('should handle working hours that cross midnight (e.g., 05:00 → 00:30)', () => {
const cfg = fakeWorkingHoursConfig('05:00', '00:30');
const mkTs = (h, m = 0) => {
const d = new Date();
d.setHours(h);
d.setMinutes(m);
d.setSeconds(0);
d.setMilliseconds(0);
return d.getTime();
};
expect(duringWorkingHoursOrNotSet(cfg, mkTs(23, 0))).to.be.true; // 23:00 => within window
expect(duringWorkingHoursOrNotSet(cfg, mkTs(1, 0))).to.be.false; // 01:00 => outside window
expect(duringWorkingHoursOrNotSet(cfg, mkTs(6, 0))).to.be.true; // 06:00 => within window
});
});
});

View File

@@ -18,11 +18,14 @@ import Jobs from './views/jobs/Jobs';
import './App.less';
import TrackingModal from './components/tracking/TrackingModal.jsx';
import { Banner } from '@douyinfe/semi-ui';
import VersionBanner from './components/version/VersionBanner.jsx';
import Listings from './views/listings/Listings.jsx';
export default function FredyApp() {
const actions = useActions();
const [loading, setLoading] = React.useState(true);
const currentUser = useSelector((state) => state.user.currentUser);
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
const settings = useSelector((state) => state.generalSettings.settings);
useEffect(() => {
@@ -34,6 +37,7 @@ export default function FredyApp() {
await actions.jobs.getProcessingTimes();
await actions.notificationAdapter.getAdapter();
await actions.generalSettings.getGeneralSettings();
await actions.versionUpdate.getVersionUpdate();
}
setLoading(false);
}
@@ -47,22 +51,18 @@ export default function FredyApp() {
const isAdmin = () => currentUser != null && currentUser.isAdmin;
const login = () => (
return loading ? null : needsLogin() ? (
<Routes>
<Route path="/login" element={<Login />} />
<Route path="*" element={<Navigate to="/login" replace />} />
</Routes>
);
return loading ? null : needsLogin() ? (
login()
) : (
<div className="app">
<div className="app__container">
<Logout />
<Logo width={190} white />
<Menu isAdmin={isAdmin()} />
{versionUpdate?.newVersion && <VersionBanner />}
{settings.demoMode && (
<>
<Banner
@@ -82,6 +82,7 @@ export default function FredyApp() {
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
<Route path="/jobs/insights/:jobId" element={<JobInsight />} />
<Route path="/jobs" element={<Jobs />} />
<Route path="/listings" element={<Listings />} />
{/* Permission-aware routes */}
<Route

BIN
ui/src/assets/no_image.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 242 KiB

View File

@@ -5,5 +5,5 @@ import logoWhite from '../../assets/logo_white.png';
import './Logo.less';
export default function Logo({ width = 350, white = false } = {}) {
return <img src={white ? logoWhite : logo} width={width} className="logo" />;
return <img src={white ? logoWhite : logo} width={width} className="logo" alt="Fredy Logo" />;
}

View File

@@ -3,7 +3,7 @@ import { useNavigate } from 'react-router-dom';
import { Tabs, TabPane } from '@douyinfe/semi-ui';
import { useLocation } from 'react-router-dom';
import { IconUser, IconTerminal, IconSetting } from '@douyinfe/semi-icons';
import { IconUser, IconTerminal, IconSetting, IconArchive } from '@douyinfe/semi-icons';
import './Menu.less';
function parsePathName(name) {
@@ -25,6 +25,15 @@ const TopMenu = function TopMenu({ isAdmin }) {
</span>
}
/>
<TabPane
itemKey="/listings"
tab={
<span>
<IconArchive />
Found listings
</span>
}
/>
{isAdmin && (
<TabPane
@@ -44,7 +53,7 @@ const TopMenu = function TopMenu({ isAdmin }) {
tab={
<span>
<IconSetting />
General
Settings
</span>
}
/>

View File

@@ -32,7 +32,7 @@ export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged,
dataIndex: 'name',
},
{
title: 'Findings',
title: 'Listings',
dataIndex: 'numberOfFoundListings',
render: (value) => {
return value || 0;

View File

@@ -0,0 +1,184 @@
import React, { useState, useEffect, useMemo } from 'react';
import { Table, Popover, Input, Descriptions, Tag, Image } from '@douyinfe/semi-ui';
import { useActions, useSelector } from '../../services/state/store.js';
import { IconClose, IconSearch, IconTick } from '@douyinfe/semi-icons';
import * as timeService from '../../services/time/timeService.js';
import debounce from 'lodash/debounce';
import no_image from '../../assets/no_image.jpg';
import './ListingsTable.less';
import { format } from '../../services/time/timeService.js';
const columns = [
{
title: '#',
dataIndex: 'is_active',
width: 58,
sorter: true,
render: (value) => {
return value ? (
<div style={{ color: 'rgba(var(--semi-green-6), 1)' }}>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Listing still online"
>
<IconTick />
</Popover>
</div>
) : (
<div style={{ color: 'rgba(var(--semi-red-5), 1)' }}>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Listing not online anymore"
>
<IconClose />
</Popover>
</div>
);
},
},
{
title: 'Job-Name',
sorter: true,
dataIndex: 'job_name',
width: 170,
},
{
title: 'Listing date',
width: 130,
dataIndex: 'created_at',
sorter: true,
render: (text) => timeService.format(text),
},
{
title: 'Provider',
width: 130,
dataIndex: 'provider',
sorter: true,
render: (text) => text.charAt(0).toUpperCase() + text.slice(1),
},
{
title: 'Price',
width: 100,
dataIndex: 'price',
sorter: true,
render: (text) => text + ' €',
},
{
title: 'Address',
width: 150,
dataIndex: 'address',
sorter: true,
},
{
title: 'Title',
dataIndex: 'title',
sorter: true,
render: (text, row) => {
return (
<a href={row.url} target="_blank" rel="noopener noreferrer">
{text}
</a>
);
},
},
];
export default function ListingsTable() {
const tableData = useSelector((state) => state.listingsTable);
const actions = useActions();
const [page, setPage] = useState(1);
const pageSize = 15;
const [sortData, setSortData] = useState({});
const [filter, setFilter] = useState(null);
const handlePageChange = (_page) => {
setPage(_page);
};
useEffect(() => {
let sortfield = null;
let sortdir = null;
if (sortData != null && Object.keys(sortData).length > 0) {
sortfield = sortData.field;
sortdir = sortData.direction;
}
actions.listingsTable.getListingsTable({ page, pageSize, sortfield, sortdir, filter });
}, [page, sortData, filter]);
const handleFilterChange = useMemo(() => debounce((value) => setFilter(value), 500), []);
const expandRowRender = (record) => {
return (
<div className="listingsTable__expanded">
<div>
{record.image_url == null ? (
<Image height={200} src={no_image} />
) : (
<Image height={200} src={record.image_url} />
)}
</div>
<div>
<Descriptions align="justify">
<Descriptions.Item itemKey="Listing still online">
<Tag size="small" shape="circle" color={record.is_active ? 'green' : 'red'}>
{record.is_active ? 'Yes' : 'No'}
</Tag>
</Descriptions.Item>
<Descriptions.Item itemKey="Link">
<a href={record.link} target="_blank" rel="noreferrer">
Link to Listing
</a>
</Descriptions.Item>
<Descriptions.Item itemKey="Listing date">{format(record.created_at)}</Descriptions.Item>
<Descriptions.Item itemKey="Price">{record.price} </Descriptions.Item>
</Descriptions>
<b>{record.title}</b>
<p>{record.description == null ? 'No description available' : record.description}</p>
</div>
</div>
);
};
return (
<div>
<Input
prefix={<IconSearch />}
showClear
className="listingsTable__search"
placeholder="Search"
onChange={handleFilterChange}
/>
<Table
rowKey="id"
hideExpandedColumn={false}
sticky={{ top: 5 }}
columns={columns}
expandedRowRender={expandRowRender}
dataSource={tableData?.result || []}
onChange={(changeSet) => {
if (changeSet?.extra?.changeType === 'sorter') {
setSortData({
field: changeSet.sorter.dataIndex,
direction: changeSet.sorter.sortOrder === 'ascend' ? 'asc' : 'desc',
});
}
}}
pagination={{
currentPage: page,
//for now fixed
pageSize,
total: tableData?.totalNumber || 0,
onPageChange: handlePageChange,
}}
/>
</div>
);
}

View File

@@ -0,0 +1,10 @@
.listingsTable {
&__search {
margin-bottom: 1rem !important;
}
&__expanded {
display: flex;
gap: 1rem;
}
}

View File

@@ -0,0 +1,37 @@
import React from 'react';
import { Banner, Descriptions } from '@douyinfe/semi-ui';
import { useSelector } from '../../services/state/store.js';
import { MarkdownRender } from '@douyinfe/semi-ui';
import './VersionBanner.less';
export default function VersionBanner() {
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
return (
<Banner
className="versionBanner"
type="success"
icon={null}
description={
<div style={{ overflow: 'auto' }}>
<p>A new version of Fredy is available. Update now to take advantage of the latest features and bug fixes.</p>
<Descriptions row size="small">
<Descriptions.Item itemKey="Your Version">{versionUpdate.localFredyVersion}</Descriptions.Item>
<Descriptions.Item itemKey="Latest Version">{versionUpdate.version}</Descriptions.Item>
<Descriptions.Item itemKey="Github Release">
<a href={versionUpdate.url} target="_blank" rel="noreferrer">
{versionUpdate.url}
</a>{' '}
</Descriptions.Item>
</Descriptions>
<p>
<b>
<small>Release Notes</small>
</b>
</p>
<MarkdownRender raw={versionUpdate.body} style={{ height: '200px' }} />
</div>
}
/>
);
}

View File

@@ -0,0 +1,3 @@
.versionBanner {
margin-bottom: 1rem;
}

View File

@@ -4,6 +4,7 @@
import { create } from 'zustand';
import { shallow } from 'zustand/shallow';
import { xhrGet } from '../xhr.js';
import queryString from 'query-string';
const logger = (config) => (set, get, api) =>
config(
@@ -118,13 +119,50 @@ export const useFredyState = create(
}
},
},
versionUpdate: {
async getVersionUpdate() {
try {
const response = await xhrGet('/api/version');
set((state) => ({
versionUpdate: { ...state.versionUpdate, versionUpdate: response.json },
}));
} catch (Exception) {
console.error('Error while trying to get resource for api/version. Error:', Exception);
}
},
},
listingsTable: {
async getListingsTable({ page = 1, pageSize = 20, filter = null, sortfield = null, sortdir = 'asc' }) {
try {
const qryString = queryString.stringify({
page,
pageSize,
filter,
sortfield,
sortdir,
});
const response = await xhrGet(`/api/listings/table?${qryString}`);
set((state) => ({
listingsTable: { ...state.listingsTable, ...response.json },
}));
} catch (Exception) {
console.error('Error while trying to get resource for api/listings. Error:', Exception);
}
},
},
};
// Initial state
const initial = {
notificationAdapter: [],
listingsTable: {
totalNumber: 0,
page: 1,
result: [],
},
generalSettings: { settings: {} },
demoMode: { demoMode: false },
versionUpdate: {},
provider: [],
jobs: { jobs: [], insights: {}, processingTimes: {} },
user: { users: [], currentUser: null },
@@ -135,6 +173,8 @@ export const useFredyState = create(
notificationAdapter: { ...effects.notificationAdapter },
generalSettings: { ...effects.generalSettings },
demoMode: { ...effects.demoMode },
versionUpdate: { ...effects.versionUpdate },
listingsTable: { ...effects.listingsTable },
provider: { ...effects.provider },
jobs: { ...effects.jobs },
user: { ...effects.user },

View File

@@ -186,7 +186,7 @@ const GeneralSettings = function GeneralSettings() {
<Divider margin="1rem" />
<SegmentPart
name="Working hours"
helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
helpText="During these hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
Icon={IconCalendar}
>
<div className="generalSettings__timePickerContainer">

View File

@@ -89,7 +89,7 @@ export default function JobMutator() {
/>
)}
<Headline text={jobToBeEdit ? 'Edit a Job' : 'Create a new Job'} />
<Headline text={jobToBeEdit ? 'Edit Job' : 'Create new Job'} />
<form>
<SegmentPart name="Name">
<Input

View File

@@ -0,0 +1,11 @@
import React from 'react';
import ListingsTable from '../../components/table/ListingsTable.jsx';
export default function Listings() {
return (
<div>
<ListingsTable />
</div>
);
}

231
yarn.lock
View File

@@ -1203,10 +1203,10 @@
minimatch "^3.1.2"
strip-json-comments "^3.1.1"
"@eslint/js@9.35.0":
version "9.35.0"
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.35.0.tgz#ffbc7e13cf1204db18552e9cd9d4a8e17c692d07"
integrity sha512-30iXE9whjlILfWobBkNerJo+TXYsgVM5ERQwMcMKCHckHflCmf7wXDAHlARoWnh0s1U72WqlbeyE7iAcCzuCPw==
"@eslint/js@9.36.0":
version "9.36.0"
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.36.0.tgz#b1a3893dd6ce2defed5fd49de805ba40368e8fef"
integrity sha512-uhCbYtYynH30iZErszX78U+nR3pJU3RHGQ57NXy5QupD4SBVwDeU8TNBy+MjMngc1UyIW9noKqsRqfjQTBU2dw==
"@eslint/object-schema@^2.1.6":
version "2.1.6"
@@ -1548,10 +1548,10 @@
dependencies:
deepmerge "^4.2.2"
"@sendgrid/mail@8.1.5":
version "8.1.5"
resolved "https://registry.yarnpkg.com/@sendgrid/mail/-/mail-8.1.5.tgz#995ef96aaf4664d2f059ec6ca38f79f724d350f2"
integrity sha512-W+YuMnkVs4+HA/bgfto4VHKcPKLc7NiZ50/NH2pzO6UHCCFuq8/GNB98YJlLEr/ESDyzAaDr7lVE7hoBwFTT3Q==
"@sendgrid/mail@8.1.6":
version "8.1.6"
resolved "https://registry.yarnpkg.com/@sendgrid/mail/-/mail-8.1.6.tgz#9c253c13d49867fdb6f7df1360643825236eef22"
integrity sha512-/ZqxUvKeEztU9drOoPC/8opEPOk+jLlB2q4+xpx6HVLq6aFu3pMpalkTpAQz8XfRfpLp8O25bh6pGPcHDCYpqg==
dependencies:
"@sendgrid/client" "^8.1.5"
"@sendgrid/helpers" "^8.0.0"
@@ -1729,24 +1729,24 @@
resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.0.tgz#d06bbb384ebcf6c505fde1c3d0ed4ddffe0aaff8"
integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==
"@visactor/react-vchart@^2.0.4":
version "2.0.4"
resolved "https://registry.yarnpkg.com/@visactor/react-vchart/-/react-vchart-2.0.4.tgz#221760d3c9707fcee9e94b3b0fd0371540d40db0"
integrity sha512-dN0VHEXMF1QTA9JAaV1kZYxajxwwPBpMhLB1vXgY9u41prDFYyboQ7atwweyBB/xSdRdsuQgzYU/SSM/R2gNeg==
"@visactor/react-vchart@^2.0.5":
version "2.0.5"
resolved "https://registry.yarnpkg.com/@visactor/react-vchart/-/react-vchart-2.0.5.tgz#1eb3339b662f623c08cf20f57c2507760c784468"
integrity sha512-D3dAPASde1zuZiorx32jkRe9cMuc9PO3IVurw0Sm/XBzrdQE2MnoLONfM2ktT/BJQggBZaHE6+n8inGE24JyJg==
dependencies:
"@visactor/vchart" "2.0.4"
"@visactor/vchart-extension" "2.0.4"
"@visactor/vchart" "2.0.5"
"@visactor/vchart-extension" "2.0.5"
"@visactor/vrender-core" "1.0.13"
"@visactor/vrender-kits" "1.0.13"
"@visactor/vutils" "~1.0.6"
react-is "^18.2.0"
"@visactor/vchart-extension@2.0.4":
version "2.0.4"
resolved "https://registry.yarnpkg.com/@visactor/vchart-extension/-/vchart-extension-2.0.4.tgz#8ac5e138bc410d9e9b23bb3e60547f01df48bac9"
integrity sha512-KmoeI7nxpfu8vGnn86O9szjoWTtvAomBtUwdtg+cNYkX/EGxZ4LUZLe0lELSpUecRk1aqZxzdeBSFB1wQpNYRw==
"@visactor/vchart-extension@2.0.5":
version "2.0.5"
resolved "https://registry.yarnpkg.com/@visactor/vchart-extension/-/vchart-extension-2.0.5.tgz#3c023ebd56bc26531f20c2ad147e45d1fcba67ef"
integrity sha512-GG5cwtJ3wv4/DUM4/RVF7qi6WXRZyDRIv+U0WgWCYAdANINW95egJ3P+NHdcdLhA7VEdAXPde6XFSWOawcK4oQ==
dependencies:
"@visactor/vchart" "2.0.4"
"@visactor/vchart" "2.0.5"
"@visactor/vdataset" "~1.0.6"
"@visactor/vlayouts" "~1.0.6"
"@visactor/vrender-animate" "1.0.13"
@@ -1767,10 +1767,10 @@
resolved "https://registry.yarnpkg.com/@visactor/vchart-theme-utils/-/vchart-theme-utils-1.12.2.tgz#bad0035e79dabbe80890bbd6196668551a12c874"
integrity sha512-PkgSAivtUZukCWVUGCXxKcbTzI/oMj1Ky22VYcVs/KM4VFmmCywU2xjBBe1du0LUey6CAKB7bMlj5bL2jctG0A==
"@visactor/vchart@2.0.4", "@visactor/vchart@^2.0.4":
version "2.0.4"
resolved "https://registry.yarnpkg.com/@visactor/vchart/-/vchart-2.0.4.tgz#36770240ae6ffd84fa285b7610192f2e06a56299"
integrity sha512-/NWBQFYd5A52I8Bkp+iod2LAhBo4cQcxt+xazrmJ/5L17Gk/LdUqCRpnF5dk3XncHb4ls+SRNGkH4kf0rNH2Mg==
"@visactor/vchart@2.0.5", "@visactor/vchart@^2.0.5":
version "2.0.5"
resolved "https://registry.yarnpkg.com/@visactor/vchart/-/vchart-2.0.5.tgz#a7041a1fe6df5125ca02ac55946b0211f4e649ed"
integrity sha512-7emhEFGEhUZC8n/PkscVQeJn/yd4757wrta1avMHUKBVY7x9qEWYSFypXT2LJTxjTePB//dqZYE/aPy/plGWNQ==
dependencies:
"@visactor/vdataset" "~1.0.6"
"@visactor/vlayouts" "~1.0.6"
@@ -1780,7 +1780,7 @@
"@visactor/vrender-kits" "1.0.13"
"@visactor/vscale" "~1.0.6"
"@visactor/vutils" "~1.0.6"
"@visactor/vutils-extension" "2.0.4"
"@visactor/vutils-extension" "2.0.5"
"@visactor/vdataset@~1.0.6":
version "1.0.9"
@@ -1869,10 +1869,10 @@
dependencies:
"@visactor/vutils" "1.0.9"
"@visactor/vutils-extension@2.0.4":
version "2.0.4"
resolved "https://registry.yarnpkg.com/@visactor/vutils-extension/-/vutils-extension-2.0.4.tgz#a369192d0ca5dd9748a21a5f1f6eb3ea094cac6c"
integrity sha512-Q0nDVTCLeCbAi8AAj8wAZfzfZDDsYF7xXhuLjjGPrPTuItPG/fHuw/rw6yDFvdhb4XGaPwv0MaUYNPFoOl60GQ==
"@visactor/vutils-extension@2.0.5":
version "2.0.5"
resolved "https://registry.yarnpkg.com/@visactor/vutils-extension/-/vutils-extension-2.0.5.tgz#7c713c6c2bdced9c7ab599d5444b37c80ce8f8c7"
integrity sha512-qQpaANT+AtOQoQAN64qhQQXqhOo9Fn5t+hmih0pFxIye+61yEj3xUSM2GxQF6ubjqCI6DvRG0DaVw0rdcoqbGg==
dependencies:
"@visactor/vdataset" "~1.0.6"
"@visactor/vutils" "~1.0.6"
@@ -1966,7 +1966,7 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0:
dependencies:
color-convert "^2.0.1"
ansi-styles@^6.0.0, ansi-styles@^6.1.0, ansi-styles@^6.2.1:
ansi-styles@^6.1.0, ansi-styles@^6.2.1:
version "6.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5"
integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==
@@ -2197,10 +2197,10 @@ basic-ftp@^5.0.2:
resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.0.5.tgz#14a474f5fffecca1f4f406f1c26b18f800225ac0"
integrity sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==
better-sqlite3@^12.2.0:
version "12.2.0"
resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.2.0.tgz#de7c3466074f2d1a5d260f510647e822e42684d2"
integrity sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ==
better-sqlite3@^12.4.1:
version "12.4.1"
resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.4.1.tgz#f78df6c80530d1a0b750b538033e6199b7d30d26"
integrity sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ==
dependencies:
bindings "^1.5.0"
prebuild-install "^7.1.1"
@@ -2375,11 +2375,6 @@ chalk@^4.0.0, chalk@^4.1.0:
ansi-styles "^4.1.0"
supports-color "^7.1.0"
chalk@^5.6.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.6.0.tgz#a1a8d294ea3526dbb77660f12649a08490e33ab8"
integrity sha512-46QrSQFyVSEyYAgQ22hQ+zDa60YHA4fBstHmtSApj1Y5vKtG27fWowW03jCk5KcbXEWPZUIR894aARCA/G1kfQ==
character-entities-html4@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-2.1.0.tgz#1f1adb940c971a4b22ba39ddca6b618dc6e56b2b"
@@ -2456,10 +2451,10 @@ chownr@^1.1.1:
resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
chromium-bidi@8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-8.0.0.tgz#d73c9beed40317adf2bcfeb9a47087003cd467ec"
integrity sha512-d1VmE0FD7lxZQHzcDUCKZSNRtRwISXDsdg4HjdTR5+Ll5nQ/vzU12JeNmupD6VWffrPSlrnGhEWlLESKH3VO+g==
chromium-bidi@9.1.0:
version "9.1.0"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-9.1.0.tgz#356eaea018eecc7977644305ee9fd27874b2b676"
integrity sha512-rlUzQ4WzIAWdIbY/viPShhZU2n21CxDUgazXVbw4Hu1MwaeUSEksSeM6DqPgpRjCLXRk702AVRxJxoOz0dw4OA==
dependencies:
mitt "^3.0.1"
zod "^3.24.1"
@@ -2476,13 +2471,13 @@ cli-cursor@^5.0.0:
dependencies:
restore-cursor "^5.0.0"
cli-truncate@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-4.0.0.tgz#6cc28a2924fee9e25ce91e973db56c7066e6172a"
integrity sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==
cli-truncate@^5.0.0:
version "5.1.0"
resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-5.1.0.tgz#bb12607a62f0e4bb91a54aa4653b23347900bb55"
integrity sha512-7JDGG+4Zp0CsknDCedl0DYdaeOhc46QNpXi3NLQblkZpXXgA6LncLDUUyvrjSvZeF3VRQa+KiMGomazQrC1V8g==
dependencies:
slice-ansi "^5.0.0"
string-width "^7.0.0"
slice-ansi "^7.1.0"
string-width "^8.0.0"
cliui@^8.0.1:
version "8.0.1"
@@ -2548,10 +2543,10 @@ commander@2:
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
commander@^14.0.0:
version "14.0.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.0.tgz#f244fc74a92343514e56229f16ef5c5e22ced5e9"
integrity sha512-2uM9rYjPvyq39NwLRqaiLtWHyDC1FvryJDa2ATTVims5YAS4PupsEQsDvP14FqhFr0P49CYDugi59xaxJlTXRA==
commander@^14.0.1:
version "14.0.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.1.tgz#2f9225c19e6ebd0dc4404dd45821b2caa17ea09b"
integrity sha512-2JkV3gUZUVrbNA+1sjBOYLsMZ5cEEl8GTFP2a4AVz5hvasAMCQ1D2l2le/cX+pV4N6ZU17zjUahLpIXRrnWL8A==
compute-scroll-into-view@^1.0.20:
version "1.0.20"
@@ -3281,10 +3276,10 @@ eslint-visitor-keys@^4.2.1:
resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz#4cfea60fe7dd0ad8e816e1ed026c1d5251b512c1"
integrity sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==
eslint@9.35.0:
version "9.35.0"
resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.35.0.tgz#7a89054b7b9ee1dfd1b62035d8ce75547773f47e"
integrity sha512-QePbBFMJFjgmlE+cXAlbHZbHpdFVS2E/6vzCy7aKlebddvl1vadiC4JFV5u/wqTkNUwEV8WrQi257jf5f06hrg==
eslint@9.36.0:
version "9.36.0"
resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.36.0.tgz#9cc5cbbfb9c01070425d9bfed81b4e79a1c09088"
integrity sha512-hB4FIzXovouYzwzECDcUkJ4OcfOEkXTv2zRY6B9bkwjx/cprAq0uvm1nl7zvQ0/TsUk0zQiN4uPfJpB9m+rPMQ==
dependencies:
"@eslint-community/eslint-utils" "^4.8.0"
"@eslint-community/regexpp" "^4.12.1"
@@ -3292,7 +3287,7 @@ eslint@9.35.0:
"@eslint/config-helpers" "^0.3.1"
"@eslint/core" "^0.15.2"
"@eslint/eslintrc" "^3.3.1"
"@eslint/js" "9.35.0"
"@eslint/js" "9.36.0"
"@eslint/plugin-kit" "^0.3.5"
"@humanfs/node" "^0.16.6"
"@humanwhocodes/module-importer" "^1.0.1"
@@ -3721,6 +3716,11 @@ get-east-asian-width@^1.0.0:
resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz#21b4071ee58ed04ee0db653371b55b4299875389"
integrity sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==
get-east-asian-width@^1.3.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz#9bc4caa131702b4b61729cb7e42735bc550c9ee6"
integrity sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==
get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.2.7, get-intrinsic@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01"
@@ -4222,11 +4222,6 @@ is-fullwidth-code-point@^3.0.0:
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
is-fullwidth-code-point@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz#fae3167c729e7463f8461ce512b080a49268aa88"
integrity sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==
is-fullwidth-code-point@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-5.0.0.tgz#9609efced7c2f97da7b60145ef481c787c7ba704"
@@ -4559,38 +4554,30 @@ levn@^0.4.1:
prelude-ls "^1.2.1"
type-check "~0.4.0"
lilconfig@^3.1.3:
version "3.1.3"
resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-3.1.3.tgz#a1bcfd6257f9585bf5ae14ceeebb7b559025e4c4"
integrity sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==
lines-and-columns@^1.1.6:
version "1.2.4"
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
lint-staged@16.1.6:
version "16.1.6"
resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-16.1.6.tgz#b0830df339a71f4207979a47c7be8ab0f38543ad"
integrity sha512-U4kuulU3CKIytlkLlaHcGgKscNfJPNTiDF2avIUGFCv7K95/DCYQ7Ra62ydeRWmgQGg9zJYw2dzdbztwJlqrow==
lint-staged@16.2.1:
version "16.2.1"
resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-16.2.1.tgz#bb82da8ce10059296b220f321980f0ee1ce40c28"
integrity sha512-KMeYmH9wKvHsXdUp+z6w7HN3fHKHXwT1pSTQTYxB9kI6ekK1rlL3kLZEoXZCppRPXFK9PFW/wfQctV7XUqMrPQ==
dependencies:
chalk "^5.6.0"
commander "^14.0.0"
debug "^4.4.1"
lilconfig "^3.1.3"
listr2 "^9.0.3"
commander "^14.0.1"
listr2 "^9.0.4"
micromatch "^4.0.8"
nano-spawn "^1.0.2"
nano-spawn "^1.0.3"
pidtree "^0.6.0"
string-argv "^0.3.2"
yaml "^2.8.1"
listr2@^9.0.3:
version "9.0.3"
resolved "https://registry.yarnpkg.com/listr2/-/listr2-9.0.3.tgz#5181284019e1d577dc2d705ca6d3a148cf15adf3"
integrity sha512-0aeh5HHHgmq1KRdMMDHfhMWQmIT/m7nRDTlxlFqni2Sp0had9baqsjJRvDGdlvgd6NmPE0nPloOipiQJGFtTHQ==
listr2@^9.0.4:
version "9.0.4"
resolved "https://registry.yarnpkg.com/listr2/-/listr2-9.0.4.tgz#2916e633ae6e09d1a3f981172937ac1c5a8fa64f"
integrity sha512-1wd/kpAdKRLwv7/3OKC8zZ5U8e/fajCfWMxacUvB79S5nLrYGPtUI/8chMQhn3LQjsRVErTb9i1ECAwW0ZIHnQ==
dependencies:
cli-truncate "^4.0.0"
cli-truncate "^5.0.0"
colorette "^2.0.20"
eventemitter3 "^5.0.1"
log-update "^6.1.0"
@@ -5414,15 +5401,15 @@ ms@^2.1.1, ms@^2.1.3:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
nano-spawn@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/nano-spawn/-/nano-spawn-1.0.2.tgz#9853795681f0e96ef6f39104c2e4347b6ba79bf6"
integrity sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg==
nano-spawn@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/nano-spawn/-/nano-spawn-1.0.3.tgz#ef8d89a275eebc8657e67b95fc312a6527a05b8d"
integrity sha512-jtpsQDetTnvS2Ts1fiRdci5rx0VYws5jGyC+4IYOTnIQ/wwdf6JdomlHBwqC3bJYOvaKu0C2GSZ1A60anrYpaA==
nanoid@5.1.5:
version "5.1.5"
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-5.1.5.tgz#f7597f9d9054eb4da9548cdd53ca70f1790e87de"
integrity sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==
nanoid@5.1.6:
version "5.1.6"
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-5.1.6.tgz#30363f664797e7d40429f6c16946d6bd7a3f26c9"
integrity sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==
nanoid@^3.3.11:
version "3.3.11"
@@ -5975,13 +5962,13 @@ punycode@^2.1.0:
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"
integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==
puppeteer-core@24.22.0:
version "24.22.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.22.0.tgz#4d576b1a2b7699c088d3f0e843c32d81df82c3a6"
integrity sha512-oUeWlIg0pMz8YM5pu0uqakM+cCyYyXkHBxx9di9OUELu9X9+AYrNGGRLK9tNME3WfN3JGGqQIH3b4/E9LGek/w==
puppeteer-core@24.22.3:
version "24.22.3"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.22.3.tgz#63285a37da6e2c44069c0b31f2171f8ab81bbe23"
integrity sha512-M/Jhg4PWRANSbL/C9im//Yb55wsWBS5wdp+h59iwM+EPicVQQCNs56iC5aEAO7avfDPRfxs4MM16wHjOYHNJEw==
dependencies:
"@puppeteer/browsers" "2.10.10"
chromium-bidi "8.0.0"
chromium-bidi "9.1.0"
debug "^4.4.3"
devtools-protocol "0.0.1495869"
typed-query-selector "^2.12.0"
@@ -6035,16 +6022,16 @@ puppeteer-extra@^3.3.6:
debug "^4.1.1"
deepmerge "^4.2.2"
puppeteer@^24.22.0:
version "24.22.0"
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.22.0.tgz#9f6905e9c3d5c316c364adb598903a1dfbfe800f"
integrity sha512-QabGIvu7F0hAMiKGHZCIRHMb6UoH0QAJA2OaqxEU2tL5noXPrxUcotg2l3ttOA4p1PFnVIGkr6PXRAWlM2evVQ==
puppeteer@^24.22.3:
version "24.22.3"
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.22.3.tgz#07dcfabdb4e924b014cb7b96bcc92f43086e637e"
integrity sha512-mnhXzIqSYSJ1SMv1RYH07YMzWP81xCmmQj91Q8iQMZqnf97eVzeHgsGL6kpywiGCi+nQafta/+NkwM4URMy/XQ==
dependencies:
"@puppeteer/browsers" "2.10.10"
chromium-bidi "8.0.0"
chromium-bidi "9.1.0"
cosmiconfig "^9.0.0"
devtools-protocol "0.0.1495869"
puppeteer-core "24.22.0"
puppeteer-core "24.22.3"
typed-query-selector "^2.12.0"
qs@^6.14.0:
@@ -6134,17 +6121,17 @@ react-resizable@^3.0.5:
prop-types "15.x"
react-draggable "^4.0.3"
react-router-dom@7.9.1:
version "7.9.1"
resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.9.1.tgz#48044923701773da6362f9003ec46f308f293f15"
integrity sha512-U9WBQssBE9B1vmRjo9qTM7YRzfZ3lUxESIZnsf4VjR/lXYz9MHjvOxHzr/aUm4efpktbVOrF09rL/y4VHa8RMw==
react-router-dom@7.9.2:
version "7.9.2"
resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.9.2.tgz#2bb35d226ca23329f4e39c8f86d1db26ee4fdf26"
integrity sha512-pagqpVJnjZOfb+vIM23eTp7Sp/AAJjOgaowhP1f1TWOdk5/W8Uk8d/M/0wfleqx7SgjitjNPPsKeCZE1hTSp3w==
dependencies:
react-router "7.9.1"
react-router "7.9.2"
react-router@7.9.1:
version "7.9.1"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.9.1.tgz#b227410c31f24dd416c939ca5d0f8d5c8a1404d4"
integrity sha512-pfAByjcTpX55mqSDGwGnY9vDCpxqBLASg0BMNAuMmpSGESo/TaOUG6BllhAtAkCGx8Rnohik/XtaqiYUJtgW2g==
react-router@7.9.2:
version "7.9.2"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.9.2.tgz#f424a14f87e4d7b5b268ce3647876e9504e4fca6"
integrity sha512-i2TPp4dgaqrOqiRGLZmqh2WXmbdFknUyiCRmSKs0hf6fWXkTKg5h56b+9F22NbGRAMxjYfqQnpi63egzD2SuZA==
dependencies:
cookie "^1.0.1"
set-cookie-parser "^2.6.0"
@@ -6755,14 +6742,6 @@ slack@11.0.2:
dependencies:
tiny-json-http "^7.0.2"
slice-ansi@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-5.0.0.tgz#b73063c57aa96f9cd881654b15294d95d285c42a"
integrity sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==
dependencies:
ansi-styles "^6.0.0"
is-fullwidth-code-point "^4.0.0"
slice-ansi@^7.1.0:
version "7.1.0"
resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-7.1.0.tgz#cd6b4655e298a8d1bdeb04250a433094b347b9a9"
@@ -6897,6 +6876,14 @@ string-width@^7.0.0:
get-east-asian-width "^1.0.0"
strip-ansi "^7.1.0"
string-width@^8.0.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-8.1.0.tgz#9e9fb305174947cf45c30529414b5da916e9e8d1"
integrity sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==
dependencies:
get-east-asian-width "^1.3.0"
strip-ansi "^7.1.0"
string.prototype.matchall@^4.0.12:
version "4.0.12"
resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz#6c88740e49ad4956b1332a911e949583a275d4c0"
@@ -7421,10 +7408,10 @@ vfile@^6.0.0:
"@types/unist" "^3.0.0"
vfile-message "^4.0.0"
vite@7.1.6:
version "7.1.6"
resolved "https://registry.yarnpkg.com/vite/-/vite-7.1.6.tgz#336806d29983135677f498a05efb0fd46c5eef2d"
integrity sha512-SRYIB8t/isTwNn8vMB3MR6E+EQZM/WG1aKmmIUCfDXfVvKfc20ZpamngWHKzAmmu9ppsgxsg4b2I7c90JZudIQ==
vite@7.1.7:
version "7.1.7"
resolved "https://registry.yarnpkg.com/vite/-/vite-7.1.7.tgz#ed3f9f06e21d6574fe1ad425f6b0912d027ffc13"
integrity sha512-VbA8ScMvAISJNJVbRDTJdCwqQoAareR/wutevKanhR2/1EkoXVZVkkORaYm/tNVCjP/UDTKtcw3bAkwOUdedmA==
dependencies:
esbuild "^0.25.0"
fdir "^6.5.0"