Compare commits

..

28 Commits

Author SHA1 Message Date
Christian Kellner
3e5cd97400 Listing management (#223)
* upgrading dependencies, fixing image placeholder

* improving processing times label and hide when screen width is too low

* aligning run now button

* renaming settings -> general settings

* smaller security and memory improvements

* improving footer

* preparing listing management

* improve filtering for listings

* preparing new settings page

* preparing new settings page

* storing settings in db

* next release version
2025-12-09 13:56:46 +01:00
orangecoding
5cfa674d7f adding unraid logo 2025-12-09 09:17:21 +01:00
orangecoding
5bd4219743 upgrading dependencies | adding ohneMakler provider 2025-12-08 20:31:28 +01:00
orangecoding
ea24eb4374 upgrading dependencies 2025-12-04 09:58:58 +01:00
orangecoding
9f67e30ff4 upgrade version 2025-11-27 16:09:44 +01:00
orangecoding
20d44b60ad upgrading dependencies 2025-11-27 15:54:54 +01:00
orangecoding
22df683969 more efficient bot protection 2025-11-27 10:30:47 +01:00
Robin Fuchs
4aab850b4f feat: updated the UI to enable editing of provider URLs (#234)
* feat: updated the UI to enable editing of provider URLs


---------

Co-authored-by: foxx-tech <robin.foxx.tech@gmail.com>
2025-11-26 17:10:42 +01:00
Efe
3eb3f6ee66 fix: notification adapter modal improvements (#230)
* fix: fix notification modal
2025-11-18 12:24:27 +01:00
Efe
1b2fc79536 feat: add http adapter (#231)
* feat: add http adapter
2025-11-18 12:23:50 +01:00
orangecoding
0606122736 improving bot detection prevention 2025-11-16 19:59:08 +01:00
orangecoding
53d5098cec fixing wrong number extraction 2025-11-03 20:01:55 +01:00
Christian Kellner
32c7518454 Listing improvements (#222)
* upgrading dependencies, fixing image placeholder

* improving processing times label and hide when screen width is too low

* aligning run now button

* renaming settings -> general settings

* smaller security and memory improvements

* improving footer
2025-11-01 10:46:55 +01:00
orangecoding
db3702ed33 improve markdown readme's & and adding ability to send telegram messages to a topic in a supergroup 2025-10-30 12:42:03 +01:00
orangecoding
e3c62d4696 fixing test runner 2025-10-29 10:35:07 +01:00
orangecoding
79a8420dfb improving similarity cache 2025-10-29 09:36:05 +01:00
orangecoding
d433b13db6 next release version 2025-10-12 16:47:46 +02:00
orangecoding
41d9274dfd reducing logging 2025-10-12 16:47:28 +02:00
orangecoding
0436c7f7d7 upgrading dependencies / FredyRuntime >> FredyPipeline 2025-10-12 16:43:56 +02:00
Christian Kellner
a1cb57318e Update README.md 2025-10-11 17:37:51 +02:00
orangecoding
2566db9805 improve index 2025-10-08 15:00:28 +02:00
orangecoding
b48f786fd3 improve docu 2025-10-08 12:16:10 +02:00
orangecoding
9c74129489 fixing listings 2025-10-07 21:22:29 +02:00
orangecoding
33120ebeca ability to share jobs with users 2025-10-07 21:06:59 +02:00
orangecoding
de2dd05c70 reverting docker file change 2025-10-07 07:18:45 +02:00
orangecoding
e4784e5960 reverting docker file change 2025-10-06 20:21:26 +02:00
orangecoding
2e537ce0be improving ntfy error handling 2025-10-06 20:19:53 +02:00
orangecoding
f0f1244baa using docker without root 2025-10-06 19:55:37 +02:00
89 changed files with 3047 additions and 1242 deletions

View File

@@ -19,4 +19,4 @@ jobs:
cache: 'yarn'
- run: yarn install
- run: yarn test
- run: yarn testGH

View File

@@ -11,7 +11,7 @@
<p align="center">
<a href="https://fredy.orange-coding.net/" target="_blank">Website</a>&nbsp;&nbsp;|&nbsp;&nbsp;
<a href="https://demo-fredy.orange-coding.net/" target="_blank">Demo</a>
<a href="https://fredy-demo.orange-coding.net/" target="_blank">Demo</a>
</p>
<p align="center">
@@ -202,7 +202,7 @@ flowchart TD
F2["Adapter 2"]
end
A1 --> B["FredyRuntime"]
A1 --> B["FredyPipeline"]
A2 --> B
A3 --> B
B --> C1 & C2 & C3

View File

@@ -1 +1 @@
{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}
{"sqlitepath":"/db"}

BIN
doc/unraid_fredy_logo.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 417 KiB

6
docker-test.sh Normal file → Executable file
View File

@@ -7,12 +7,12 @@ if [ "$(docker ps -aq -f name=fredy)" ]; then
docker rm fredy || true
fi
# Build image from local Dockerfile
docker build -t fredy:local .
# Build image from local Dockerfile, forcing a fresh build without cache
docker build --no-cache -t fredy:local .
# Run container with volumes and port mapping
docker run -d --name fredy \
-v fredy_conf:/conf \
-v fredy_db:/db \
-p 9998:9998 \
fredy:local
fredy:local

View File

@@ -7,11 +7,14 @@
content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"
/>
<meta name="google" content="notranslate" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<title>Fredy</title>
<title>Fredy || Real Estate Finder</title>
</head>
<body theme-mode="dark">
<div id="fredy" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0"></div>
</body>
<script type="module" src="/ui/src/Index.jsx"></script>
</html>

View File

@@ -1,9 +1,9 @@
import fs from 'fs';
import path from 'path';
import { checkIfConfigIsAccessible, config, getProviders, refreshConfig } from './lib/utils.js';
import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import * as jobStorage from './lib/services/storage/jobStorage.js';
import FredyRuntime from './lib/FredyRuntime.js';
import FredyPipeline from './lib/FredyPipeline.js';
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
@@ -12,43 +12,52 @@ import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
import logger from './lib/services/logger.js';
import { bus } from './lib/services/events/event-bus.js';
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
import { getSettings } from './lib/services/storage/settingsStorage.js';
import SqliteConnection from './lib/services/storage/SqliteConnection.js';
//in the config, we store the path of the sqlite file, thus we must check if it is available
const isConfigAccessible = await checkIfConfigIsAccessible();
await SqliteConnection.init();
// Load configuration before any other startup steps
await refreshConfig();
const isConfigAccessible = await checkIfConfigIsAccessible();
if (!isConfigAccessible) {
logger.error('Configuration exists, but is not accessible. Please check the file permission');
process.exit(1);
}
// Run DB migrations once at startup and block until finished
await runMigrations();
const settings = await getSettings();
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
const rawDir = config.sqlitepath || '/db';
const rawDir = settings.sqlitepath || '/db';
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
if (!fs.existsSync(absDir)) {
fs.mkdirSync(absDir, { recursive: true });
}
// Run DB migrations once at startup and block until finished
await runMigrations();
// Load provider modules once at startup
const providers = await getProviders();
similarityCache.initSimilarityCache();
similarityCache.startSimilarityCacheReloader();
//assuming interval is always in minutes
const INTERVAL = config.interval * 60 * 1000;
const INTERVAL = settings.interval * 60 * 1000;
// Initialize API only after migrations completed
await import('./lib/api/api.js');
if (config.demoMode) {
if (settings.demoMode) {
logger.info('Running in demo mode');
cleanupDemoAtMidnight();
}
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
ensureAdminUserExists();
ensureDemoUserExists();
@@ -62,10 +71,10 @@ bus.on('jobs:runAll', () => {
});
const execute = () => {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
if (!config.demoMode) {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(settings, Date.now());
if (!settings.demoMode) {
if (isDuringWorkingHoursOrNotSet) {
config.lastRun = Date.now();
settings.lastRun = Date.now();
jobStorage
.getJobs()
.filter((job) => job.enabled)
@@ -75,7 +84,7 @@ const execute = () => {
.forEach(async (prov) => {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyRuntime(
await new FredyPipeline(
matchedProvider.config,
job.notificationAdapter,
prov.id,

216
lib/FredyPipeline.js Executable file
View File

@@ -0,0 +1,216 @@
import { NoNewListingsWarning } from './errors.js';
import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
import * as notify from './notification/notify.js';
import Extractor from './services/extractor/extractor.js';
import urlModifier from './services/queryStringMutator.js';
import logger from './services/logger.js';
/**
* @typedef {Object} Listing
* @property {string} id Stable unique identifier (hash) of the listing.
* @property {string} title Title or headline of the listing.
* @property {string} [address] Optional address/location text.
* @property {string} [price] Optional price text/value.
* @property {string} [url] Link to the listing detail page.
* @property {any} [meta] Provider-specific additional metadata.
*/
/**
* @typedef {Object} SimilarityCache
* @property {(title:string, address?:string)=>boolean} hasSimilarEntries Returns true if a similar entry is known.
* @property {(title:string, address?:string)=>void} addCacheEntry Adds a new entry to the similarity cache.
*/
/**
* Runtime orchestrator for fetching, normalizing, filtering, deduplicating, storing,
* and notifying about new listings from a configured provider.
*
* The execution flow is:
* 1) Prepare provider URL (sorting, etc.)
* 2) Extract raw listings from the provider
* 3) Normalize listings to the provider schema
* 4) Filter out incomplete/blacklisted listings
* 5) Identify new listings (vs. previously stored hashes)
* 6) Persist new listings
* 7) Filter out entries similar to already seen ones
* 8) Dispatch notifications
*/
class FredyPipeline {
/**
* Create a new runtime instance for a single provider/job execution.
*
* @param {Object} providerConfig Provider configuration.
* @param {string} providerConfig.url Base URL to crawl.
* @param {string} [providerConfig.sortByDateParam] Query parameter used to enforce sorting by date (provider-specific).
* @param {string} [providerConfig.waitForSelector] CSS selector to wait for before parsing content.
* @param {Object.<string, string>} providerConfig.crawlFields Mapping of field names to selectors/paths to extract.
* @param {string} providerConfig.crawlContainer CSS selector for the container holding listing items.
* @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape.
* @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings.
* @param {(url:string, waitForSelector?:string)=>Promise<void>|Promise<Listing[]>} [providerConfig.getListings] Optional override to fetch listings.
*
* @param {Object} notificationConfig Notification configuration passed to notification adapters.
* @param {string} providerId The ID of the provider currently in use.
* @param {string} jobKey Key of the job that is currently running (from within the config).
* @param {SimilarityCache} similarityCache Cache instance for checking similar entries.
*/
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
this._providerConfig = providerConfig;
this._notificationConfig = notificationConfig;
this._providerId = providerId;
this._jobKey = jobKey;
this._similarityCache = similarityCache;
}
/**
* Execute the end-to-end pipeline for a single provider run.
*
* @returns {Promise<Listing[]|void>} Resolves to the list of new (and similarity-filtered) listings
* after notifications have been sent; resolves to void when there are no new listings.
*/
execute() {
return Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
.then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
.then(this._normalize.bind(this))
.then(this._filter.bind(this))
.then(this._findNew.bind(this))
.then(this._save.bind(this))
.then(this._filterBySimilarListings.bind(this))
.then(this._notify.bind(this))
.catch(this._handleError.bind(this));
}
/**
* Fetch listings from the provider, using the default Extractor flow unless
* a provider-specific getListings override is supplied.
*
* @param {string} url The provider URL to fetch from.
* @returns {Promise<Listing[]>} Resolves with an array of listings (empty when none found).
*/
_getListings(url) {
const extractor = new Extractor();
return new Promise((resolve, reject) => {
extractor
.execute(url, this._providerConfig.waitForSelector)
.then(() => {
const listings = extractor.parseResponseText(
this._providerConfig.crawlContainer,
this._providerConfig.crawlFields,
url,
);
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
logger.error(err);
});
});
}
/**
* Normalize raw listings into the provider-specific Listing shape.
*
* @param {any[]} listings Raw listing entries from the extractor or override.
* @returns {Listing[]} Normalized listings.
*/
_normalize(listings) {
return listings.map(this._providerConfig.normalize);
}
/**
* Filter out listings that are missing required fields and those rejected by the
* provider's blacklist/filter function.
*
* @param {Listing[]} listings Listings to filter.
* @returns {Listing[]} Filtered listings that pass validation and provider filter.
*/
_filter(listings) {
const keys = Object.keys(this._providerConfig.crawlFields);
const filteredListings = listings.filter((item) => keys.every((key) => key in item));
return filteredListings.filter(this._providerConfig.filter);
}
/**
* Determine which listings are new by comparing their IDs against stored hashes.
*
* @param {Listing[]} listings Listings to evaluate for novelty.
* @returns {Listing[]} New listings not seen before.
* @throws {NoNewListingsWarning} When no new listings are found.
*/
_findNew(listings) {
logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
const newListings = listings.filter((o) => !hashes.includes(o.id));
if (newListings.length === 0) {
throw new NoNewListingsWarning();
}
return newListings;
}
/**
* Send notifications for new listings using the configured notification adapter(s).
*
* @param {Listing[]} newListings New listings to notify about.
* @returns {Promise<Listing[]>} Resolves to the provided listings after notifications complete.
* @throws {NoNewListingsWarning} When there are no listings to notify about.
*/
_notify(newListings) {
if (newListings.length === 0) {
throw new NoNewListingsWarning();
}
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
return Promise.all(sendNotifications).then(() => newListings);
}
/**
* Persist new listings and pass them through.
*
* @param {Listing[]} newListings Listings to store.
* @returns {Listing[]} The same listings, unchanged.
*/
_save(newListings) {
logger.debug(`Storing ${newListings.length} new listings (Provider: '${this._providerId}')`);
storeListings(this._jobKey, this._providerId, newListings);
return newListings;
}
/**
* Remove listings that are similar to already known entries according to the similarity cache.
* Adds the remaining listings to the cache.
*
* @param {Listing[]} listings Listings to filter by similarity.
* @returns {Listing[]} Listings considered unique enough to keep.
*/
_filterBySimilarListings(listings) {
return listings.filter((listing) => {
const similar = this._similarityCache.checkAndAddEntry({
title: listing.title,
address: listing.address,
price: listing.price,
});
if (similar) {
logger.debug(
`Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
);
}
return !similar;
});
}
/**
* Handle errors occurring in the pipeline, logging levels depending on type.
*
* @param {Error} err Error instance thrown by previous steps.
* @returns {void}
*/
_handleError(err) {
if (err.name === 'NoNewListingsWarning') {
logger.debug(`No new listings found (Provider: '${this._providerId}').`);
} else {
logger.error(err);
}
}
}
export default FredyPipeline;

View File

@@ -1,127 +0,0 @@
import { NoNewListingsWarning } from './errors.js';
import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
import * as notify from './notification/notify.js';
import Extractor from './services/extractor/extractor.js';
import urlModifier from './services/queryStringMutator.js';
import logger from './services/logger.js';
class FredyRuntime {
/**
*
* @param providerConfig the config for the specific provider, we're going to query at the moment
* @param notificationConfig the config for all notifications
* @param providerId the id of the provider currently in use
* @param jobKey key of the job that is currently running (from within the config)
* @param similarityCache cache instance holding values to check for similarity of entries
*/
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
this._providerConfig = providerConfig;
this._notificationConfig = notificationConfig;
this._providerId = providerId;
this._jobKey = jobKey;
this._similarityCache = similarityCache;
}
execute() {
return (
//modify the url to make sure search order is correctly set
Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
//scraping the site and try finding new listings
.then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
//bring them in a proper form (dictated by the provider)
.then(this._normalize.bind(this))
//filter listings with stuff tagged by the blacklist of the provider
.then(this._filter.bind(this))
//check if new listings available. if so proceed
.then(this._findNew.bind(this))
//store everything in db
.then(this._save.bind(this))
//check for similar listings. if found, remove them before notifying
.then(this._filterBySimilarListings.bind(this))
//notify the user using the configured notification adapter
.then(this._notify.bind(this))
//if an error occurred on the way, handle it here.
.catch(this._handleError.bind(this))
);
}
_getListings(url) {
const extractor = new Extractor();
return new Promise((resolve, reject) => {
extractor
.execute(url, this._providerConfig.waitForSelector)
.then(() => {
const listings = extractor.parseResponseText(
this._providerConfig.crawlContainer,
this._providerConfig.crawlFields,
url,
);
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
logger.error(err);
});
});
}
_normalize(listings) {
return listings.map(this._providerConfig.normalize);
}
_filter(listings) {
//only return those where all the fields have been found
const keys = Object.keys(this._providerConfig.crawlFields);
const filteredListings = listings.filter((item) => keys.every((key) => key in item));
return filteredListings.filter(this._providerConfig.filter);
}
_findNew(listings) {
logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
const newListings = listings.filter((o) => !hashes.includes(o.id));
if (newListings.length === 0) {
throw new NoNewListingsWarning();
}
return newListings;
}
_notify(newListings) {
if (newListings.length === 0) {
throw new NoNewListingsWarning();
}
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
return Promise.all(sendNotifications).then(() => newListings);
}
_save(newListings) {
logger.debug(`Storing ${newListings.length} new listings (Provider: '${this._providerId}')`);
storeListings(this._jobKey, this._providerId, newListings);
return newListings;
}
_filterBySimilarListings(listings) {
const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
if (similar) {
logger.debug(
`Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
);
}
return !similar;
});
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
return filteredList;
}
_handleError(err) {
if (err.name === 'NoNewListingsWarning') {
logger.debug(`No new listings found (Provider: '${this._providerId}').`);
} else {
logger.error(err);
}
}
}
export default FredyRuntime;

View File

@@ -7,7 +7,6 @@ import { versionRouter } from './routes/versionRouter.js';
import { loginRouter } from './routes/loginRoute.js';
import { userRouter } from './routes/userRoute.js';
import { jobRouter } from './routes/jobRouter.js';
import { config } from '../utils.js';
import bodyParser from 'body-parser';
import restana from 'restana';
import files from 'serve-static';
@@ -16,9 +15,11 @@ import { getDirName } from '../utils.js';
import { demoRouter } from './routes/demoRouter.js';
import logger from '../services/logger.js';
import { listingsRouter } from './routes/listingsRouter.js';
import { getSettings } from '../services/storage/settingsStorage.js';
import { featureRouter } from './routes/featureRouter.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../ui/public'));
const PORT = config.port || 9998;
const PORT = (await getSettings()).port || 9998;
service.use(bodyParser.json());
service.use(cookieSession());
@@ -39,6 +40,7 @@ service.use('/api/version', versionRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
service.use('/api/listings', listingsRouter);
service.use('/api/features', featureRouter);
//this route is unsecured intentionally as it is being queried from the login page
service.use('/api/demo', demoRouter);

View File

@@ -1,10 +1,11 @@
import restana from 'restana';
import { config } from '../../utils.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const demoRouter = service.newRouter();
demoRouter.get('/', async (req, res) => {
res.body = Object.assign({}, { demoMode: config.demoMode });
const settings = await getSettings();
res.body = Object.assign({}, { demoMode: settings.demoMode });
res.send();
});

View File

@@ -0,0 +1,12 @@
import restana from 'restana';
import getFeatures from '../../features.js';
const service = restana();
const featureRouter = service.newRouter();
featureRouter.get('/', async (req, res) => {
const features = getFeatures();
res.body = Object.assign({}, { features });
res.send();
});
export { featureRouter };

View File

@@ -1,24 +1,30 @@
import restana from 'restana';
import { config, getDirName, readConfigFromStorage, refreshConfig } from '../../utils.js';
import { getDirName } from '../../utils.js';
import fs from 'fs';
import { ensureDemoUserExists } from '../../services/storage/userStorage.js';
import logger from '../../services/logger.js';
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const generalSettingsRouter = service.newRouter();
generalSettingsRouter.get('/', async (req, res) => {
res.body = Object.assign({}, config);
res.body = Object.assign({}, await getSettings());
res.send();
});
generalSettingsRouter.post('/', async (req, res) => {
const settings = req.body;
const { sqlitepath, ...appSettings } = req.body || {};
const localSettings = await getSettings();
if (localSettings.demoMode) {
res.send(new Error('In demo mode, it is not allowed to change these settings.'));
return;
}
try {
if (config.demoMode) {
res.send(new Error('In demo mode, it is not allowed to change these settings.'));
return;
if (typeof sqlitepath !== 'undefined') {
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
}
const currentConfig = await readConfigFromStorage();
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ ...currentConfig, ...settings }));
await refreshConfig();
upsertSettings(appSettings);
ensureDemoUserExists();
} catch (err) {
logger.error(err);

View File

@@ -1,10 +1,10 @@
import restana from 'restana';
import * as jobStorage from '../../services/storage/jobStorage.js';
import * as userStorage from '../../services/storage/userStorage.js';
import { config } from '../../utils.js';
import { isAdmin } from '../security.js';
import logger from '../../services/logger.js';
import { bus } from '../../services/events/event-bus.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const jobRouter = service.newRouter();
@@ -24,13 +24,30 @@ function doesJobBelongsToUser(job, req) {
jobRouter.get('/', async (req, res) => {
const isUserAdmin = isAdmin(req);
//show only the jobs which belongs to the user (or all of the user is an admin)
res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
res.body = jobStorage
.getJobs()
.filter(
(job) =>
isUserAdmin || job.userId === req.session.currentUser || job.shared_with_user.includes(req.session.currentUser),
)
.map((job) => {
return {
...job,
isOnlyShared:
!isUserAdmin &&
job.userId !== req.session.currentUser &&
job.shared_with_user.includes(req.session.currentUser),
};
});
res.send();
});
jobRouter.get('/processingTimes', async (req, res) => {
const settings = await getSettings();
res.body = {
interval: config.interval,
lastRun: config.lastRun || null,
interval: settings.interval,
lastRun: settings.lastRun || null,
};
res.send();
});
@@ -41,8 +58,15 @@ jobRouter.post('/startAll', async (req, res) => {
});
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled, shareWithUsers = [] } = req.body;
try {
let jobFromDb = jobStorage.getJob(jobId);
if (jobFromDb && !doesJobBelongsToUser(jobFromDb, req)) {
res.send(new Error('You are trying to change a job that is not associated to your user.'));
return;
}
jobStorage.upsertJob({
userId: req.session.currentUser,
jobId,
@@ -51,6 +75,7 @@ jobRouter.post('/', async (req, res) => {
blacklist,
provider,
notificationAdapter,
shareWithUsers,
});
} catch (error) {
res.send(new Error(error));
@@ -58,6 +83,7 @@ jobRouter.post('/', async (req, res) => {
}
res.send();
});
jobRouter.delete('', async (req, res) => {
const { jobId } = req.body;
try {
@@ -92,4 +118,16 @@ jobRouter.put('/:jobId/status', async (req, res) => {
}
res.send();
});
jobRouter.get('/shareableUserList', async (req, res) => {
const currentUser = req.session.currentUser;
const users = userStorage.getUsers(false);
res.body = users
.filter((user) => !user.isAdmin && user.id !== currentUser)
.map((user) => ({
id: user.id,
name: user.username,
}));
res.send();
});
export { jobRouter };

View File

@@ -1,9 +1,9 @@
import restana from 'restana';
import * as userStorage from '../../services/storage/userStorage.js';
import * as hasher from '../../services/security/hash.js';
import { config } from '../../utils.js';
import { trackDemoAccessed } from '../../services/tracking/Tracker.js';
import logger from '../../services/logger.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const loginRouter = service.newRouter();
loginRouter.get('/user', async (req, res) => {
@@ -20,6 +20,7 @@ loginRouter.get('/user', async (req, res) => {
res.send();
});
loginRouter.post('/', async (req, res) => {
const settings = await getSettings();
const { username, password } = req.body;
const user = userStorage.getUsers(true).find((user) => user.username === username);
if (user == null) {
@@ -27,7 +28,7 @@ loginRouter.post('/', async (req, res) => {
return;
}
if (user.password === hasher.hash(password)) {
if (config.demoMode) {
if (settings.demoMode) {
await trackDemoAccessed();
}

View File

@@ -1,7 +1,7 @@
import restana from 'restana';
import * as userStorage from '../../services/storage/userStorage.js';
import * as jobStorage from '../../services/storage/jobStorage.js';
import { config } from '../../utils.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const userRouter = service.newRouter();
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
@@ -11,17 +11,20 @@ function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
return req.session.currentUser === userIdToBeRemoved;
}
const nullOrEmpty = (str) => str == null || str.length === 0;
userRouter.get('/', async (req, res) => {
res.body = userStorage.getUsers(false);
res.send();
});
userRouter.get('/:userId', async (req, res) => {
const { userId } = req.params;
res.body = userStorage.getUser(userId);
res.send();
});
userRouter.delete('/', async (req, res) => {
if (config.demoMode) {
const settings = await getSettings();
if (settings.demoMode) {
res.send(new Error('In demo mode, it is not allowed to remove user.'));
return;
}
@@ -42,7 +45,8 @@ userRouter.delete('/', async (req, res) => {
res.send();
});
userRouter.post('/', async (req, res) => {
if (config.demoMode) {
const settings = await getSettings();
if (settings.demoMode) {
res.send(new Error('In demo mode, it is not allowed to change or add user.'));
return;
}

9
lib/features.js Normal file
View File

@@ -0,0 +1,9 @@
const FEATURES = {
WATCHLIST_MANAGEMENT: false,
};
export default function getFeatures() {
return {
...FEATURES,
};
}

View File

@@ -1,3 +1,8 @@
### Apprise Adapter
Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.
Use [Apprise](https://github.com/caronc/apprise-api#installation) to forward notifications to many different services.
Quick start:
- Set up an Apprise API instance (see the installation guide linked above).
- Configure your preferred notification service(s) within Apprise.
- In Fredy, point the Apprise adapter to your Apprise API endpoint.

View File

@@ -1,4 +1,3 @@
### Console Adapter
The console adapter prints everything found by Fredy into the console (not sending any notifications to you). This can be useful when you want to check if your search
criteria meet the expectations.
The console adapter prints everything found by Fredy to the console (it does not send notifications). This is useful to verify that your search criteria work as expected before enabling a real notification service.

View File

@@ -1,4 +1,8 @@
### Discord Adapter
### Discord Webhook Adapter
To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
Once you have created a webhook, copy and paste the webhook URL.
Use a Discord channel webhook to receive notifications.
Quick start:
- Create a webhook in your target Discord channel. See the "Intro to Webhooks" guide on the Discord support site: https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks
- Copy the generated webhook URL.
- In Fredy, configure the Discord adapter with this webhook URL.

View File

@@ -0,0 +1,57 @@
import { markdown2Html } from '../../services/markdown.js';
const mapListing = (listing) => ({
address: listing.address,
description: listing.description,
id: listing.id,
imageUrl: listing.image,
price: listing.price,
size: listing.size,
title: listing.title,
url: listing.link,
});
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { authToken, endpointUrl } = notificationConfig.find((a) => a.id === config.id).fields;
const listings = newListings.map(mapListing);
const body = {
jobId: jobKey,
timestamp: new Date().toISOString(),
provider: serviceName,
listings,
};
const headers = {
'Content-Type': 'application/json',
};
if (authToken != null) {
headers['Authorization'] = `Bearer ${authToken}`;
}
return fetch(endpointUrl, {
method: 'POST',
headers: headers,
body: JSON.stringify(body),
});
};
export const config = {
id: 'http',
name: 'HTTP',
readme: markdown2Html('lib/notification/adapter/http.md'),
description: 'Fredy will send a generic HTTP POST request.',
fields: {
endpointUrl: {
description: "Your application's endpoint URL.",
label: 'Endpoint URL',
type: 'text',
},
authToken: {
description: "Your application's auth token, if required by your endpoint.",
label: 'Auth token (optional)',
optional: true,
type: 'text',
},
},
};

View File

@@ -0,0 +1,43 @@
### HTTP Adapter
This is a generic adapter for sending notifications via HTTP requests.
You can leverage this adapter to integrate with various webhooks or APIs that accept HTTP requests. (e.g. Supabase
Functions, a Node.js server, etc.)
HTTP adapter supports a `authToken` field, which can be used to include an authorization token in the request headers.
Your token would be included as a Bearer token in the `Authorization` header, which is a common method for securing API requests.
Request Details:
<details>
Request Method: POST
Headers:
```
Content Type: `application/json`
Authorization: Bearer {your-optional-auth-token}
```
Body:
```json
{
"jobId": "mg1waX4RHmIzL5NDYtYp-",
"provider": "immoscout",
"timestamp": "2024-06-15T12:34:56Z",
"listings": [
{
"address": "Str. 123, Bielefeld, Germany",
"description": "Möbliert: Einziehen & wohlfühlen: Neu möbliert.",
"id": "123456789",
"imageUrl": "https://<target-url>.com/listings/123456789.jpg",
"price": "1.240 €",
"size": "38 m²",
"title": "Schöne 1-Zimmer-Wohnung in Bielefeld",
"url": "https://<target-url>.com/listings/123456789"
}
]
}
```
</details>

View File

@@ -1,8 +1,8 @@
### MailJet Adapter
### Mailjet Adapter
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
To use [Mailjet](https://mailjet.com), create an account and decide which email address Fredy should send from.
E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.
The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
For example, if you use yourGmailAccount@gmail.com, add and verify this address in Mailjet.
Provide your public/private API keys in Fredy's configuration. Fredy uses the same email template as for SendGrid.
If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).
To send to multiple recipients, separate email addresses with commas (e.g., some@email.com, someOther@email.com).

View File

@@ -1,5 +1,8 @@
### Mattermost Adapter
For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
Receive notifications in Mattermost via an incoming webhook.
As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.
Quick start:
- Create an incoming webhook following the Mattermost developer docs: https://docs.mattermost.com/developer/webhooks-incoming.html
- Copy the webhook URL.
- In Fredy, configure the Mattermost adapter with this URL and the target channel.

View File

@@ -36,7 +36,17 @@ Link: ${newListing.link}`;
method: 'POST',
headers,
body: message,
});
})
.then((res) => {
if (!res.ok) {
throw new Error(`Ntfy message could not be sent. Status code: ${res.status}`);
}
return res.text();
})
.catch((error) => {
// Ensure we reject with an Error object and prevent unhandled rejections
throw error instanceof Error ? error : new Error(String(error));
});
});
return Promise.all(promises);

View File

@@ -1,5 +1,8 @@
### ntfy Adapter
For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
Send push notifications using an ntfy topic.
As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.
Quick start:
- Create or choose a topic on your preferred ntfy instance (see docs: https://docs.ntfy.sh/publish/).
- Copy the publish URL for that topic.
- In Fredy, configure the ntfy adapter with the topic URL and set a priority.

View File

@@ -1,5 +1,8 @@
### Pushover Adapter
Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
Use Pushover to receive push notifications on your devices.
After setting up the application, please enter both your newly created User key and API token.
Setup:
- Follow Pushover's getting-started guide: https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it
- Create an application and obtain your User Key and API Token.
- In Fredy, configure the Pushover adapter with both values.

View File

@@ -1,9 +1,12 @@
### SendGrid Adapter
SendGrid is a free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
SendGrid is an email delivery service with a generous free tier, which is more than enough for Fredy.
To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.
Setup:
- Create a SendGrid account: https://sendgrid.com/
- Decide which email address Fredy should send from (e.g., yourGmailAccount@gmail.com), add it to SendGrid, and complete the verification.
- Create an API key and add it to Fredy's configuration.
- Create a Dynamic Template in SendGrid. You can copy the template from `/lib/notification/emailTemplate/template.hbs`.
Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new dynamic template. For this new template, I recommend copying and pasting the code from the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).
Sending to multiple recipients:
- Separate email addresses with commas (e.g., some@email.com, someOther@email.com).

View File

@@ -1,4 +1,5 @@
### Slack Adapter
IMPORTANT:
Don't use this adapter anymore, it is outdated and only here for backwards compatability reasons. Use the new Slack Adapter with webhooks!
### Slack Adapter (Legacy)
*IMPORTANT:*
This legacy adapter is outdated and kept only for backward compatibility. Please use the Slack adapter with webhooks instead.

View File

@@ -1,6 +1,10 @@
### Slack Adapter
### Slack Adapter (Webhooks)
IMPORTANT:
This is the new version of the Slack adapter. I strongly encourage you to use it, the old version is now unmaintained and only kept due to backwards compatability reasons.
*IMPORTANT:*
This is the recommended Slack adapter. The old Slack adapter is unmaintained and kept only for backward compatibility.
In order to use [Slack](https://slack.com), you need to create an account. When done, create a new channel and add the Webhook integration to that channel. Copy the webhook url. That's it.
Setup:
- Create a Slack account and workspace if you don't have one: https://slack.com
- Create a channel where you want to receive notifications.
- Add the Incoming Webhooks integration to that channel and copy the Webhook URL.
- In Fredy, configure the Slack Webhook adapter with this URL.

View File

@@ -1,9 +1,21 @@
### SQLite Adapter
This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. This file can be used for further analysis later.
This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. The file can be used for analysis later.
The database table contains the following columns (all stored as `TEXT` type):
The table contains the following columns (all stored as `TEXT`):
```
['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description', 'image']
```json
[
"serviceName",
"jobKey",
"id",
"size",
"rooms",
"price",
"address",
"title",
"link",
"description",
"image"
]
```

View File

@@ -117,10 +117,24 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
if (!adapterCfg || !adapterCfg.fields) {
throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`);
}
const { token, chatId } = adapterCfg.fields;
const { token, chatId, messageThreadId } = adapterCfg.fields;
if (!token || !chatId) {
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
}
// Optional Telegram topic/thread support (supergroups)
let message_thread_id;
if (messageThreadId !== undefined && messageThreadId !== null && `${messageThreadId}`.trim() !== '') {
const n = Number(messageThreadId);
if (Number.isInteger(n) && n > 0) {
message_thread_id = n;
} else {
logger.warn(
`Telegram adapter: 'messageThreadId' is invalid ('${messageThreadId}'). It must be a positive integer. Ignoring.`,
);
}
}
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
@@ -147,6 +161,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
text: buildText(jobName, serviceName, o),
parse_mode: 'HTML',
disable_web_page_preview: true,
...(message_thread_id ? { message_thread_id } : {}),
};
if (!img) {
@@ -160,6 +175,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
photo: img,
caption: buildCaption(jobName, serviceName, o),
parse_mode: 'HTML',
...(message_thread_id ? { message_thread_id } : {}),
}).catch(async (e) => {
logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
return await throttledCall('sendMessage', textPayload).catch((e) => {
@@ -174,7 +190,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
/**
* Telegram notification adapter configuration schema.
* @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string}}}}
* @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string},messageThreadId?:{type:string,label:string,description:string}}}}
*/
export const config = {
id: 'telegram',
@@ -192,5 +208,12 @@ export const config = {
label: 'Chat Id',
description: 'The chat id to send messages to you.',
},
messageThreadId: {
type: 'text',
optional: true,
label: 'Message Thread Id (optional)',
description:
'Optional: The topic/thread id within a supergroup to post into (Telegram message_thread_id). Provide a positive integer.',
},
},
};

View File

@@ -1,12 +1,55 @@
### Telegram Adapter
For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
Use this adapter to send notifications to Telegram via a bot. You will need:
- A Telegram Bot token (from BotFather)
- A chat ID (where messages will be sent)
- Optionally: a thread ID if you want to post into a specific forum topic in a group
A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.
After the user has send a message to your bot the first time, you can gather the chatId like this:
#### Create a bot
Create a bot with BotFather: open https://telegram.me/BotFather on your phone or in Telegram Desktop and follow the instructions to get your bot token.
#### Getting the chat ID
A Telegram bot cannot message a user first; you must create a conversation (or add the bot to a group/channel) so Telegram assigns a chat the bot can access.
Steps:
1. Start a chat with your bot in Telegram (or add the bot to your group/supergroup/channel) and send any message.
2. Fetch recent updates from the Bot API:
```
curl -X GET "https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates"
```
3. In the JSON response, find the message that you just sent and read `message.chat.id`. That value is your `chatId`.
- Private chats: `chat.id` is a positive number
- Groups/supergroups: `chat.id` is a negative number
Keep your bot token secret. If `getUpdates` returns an empty list, send a new message and try again, or make sure your bots privacy settings allow it to see group messages when used in groups.
#### Getting the thread ID (this is optional to be used for forum topics)
If you want messages to appear inside a specific forum topic of a supergroup with Topics enabled, you also need a thread ID. In the Telegram Bot API this is called `message_thread_id`.
When you need it:
- Required only for supergroups with Topics enabled when targeting a topic
- Not used for private chats, basic groups without Topics, or channels
Steps to obtain it:
1. In your supergroup, enable Topics (Group settings → Manage group → Topics → Enable). Now add a new topic.
2. Add your created bot to the topic. (Click on the bot and on "Add to group")
3. Open the desired topic (or create a new one) and send any message inside that topic.
4. Call `getUpdates` again:
```
curl -X GET "https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates"
```
4. In the update for the message you sent inside the topic, read `message.message_thread_id`. That number is your `threadId` for this topic.
Example (truncated):
```
curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
{
"message": {
"chat": { "id": -1001234567890, "type": "supergroup" },
"message_thread_id": 42,
"text": "hello from the topic"
}
}
```
Use `chat.id` as `chatId` and `message_thread_id` as `threadId` in your configuration.
A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
More details about bots and BotFather: https://core.telegram.org/bots#botfather

45
lib/provider/ohneMakler.js Executable file
View File

@@ -0,0 +1,45 @@
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
const link = metaInformation.baseUrl + o.link;
const id = buildHash(o.title, o.link, o.price);
return Object.assign(o, { link, id });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: 'div[data-livecomponent-id*="search/property_list"] .grid > div',
sortByDateParam: null,
waitForSelector: null,
crawlFields: {
id: 'a@href',
title: 'h4 | removeNewline | trim',
price: '.text-xl | trim',
size: 'div[title="Wohnfläche"] | trim',
address: '.text-slate-800 | removeNewline | trim',
image: 'img@src',
link: 'a@href',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
export const metaInformation = {
name: 'OhneMakler',
baseUrl: 'https://www.ohne-makler.net/immobilien',
id: 'ohneMakler',
};
export { config };

View File

@@ -8,7 +8,7 @@ function normalize(o) {
const title = o.title || 'No title available';
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
var urlReg = new RegExp(/url\((.*?)\)/gim);
const urlReg = new RegExp(/url\((.*?)\)/gim);
const image = o.image != null ? urlReg.exec(o.image)[1] : null;
return Object.assign(o, { id, address, title, link, image });
}

View File

@@ -1,8 +1,8 @@
import { removeJobsByUserId } from '../storage/jobStorage.js';
import { config } from '../../utils.js';
import { getUsers } from '../storage/userStorage.js';
import logger from '../logger.js';
import cron from 'node-cron';
import { getSettings } from '../storage/settingsStorage.js';
/**
* if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
@@ -11,12 +11,13 @@ export function cleanupDemoAtMidnight() {
cron.schedule('0 0 * * *', cleanup);
}
function cleanup() {
if (config.demoMode) {
async function cleanup() {
const settings = await getSettings();
if (settings.demoMode) {
const demoUser = getUsers(false).find((user) => user.username === 'demo');
if (demoUser == null) {
logger.error('Demo user not found, cannot remove Jobs');
return;
return Promise.resolve();
}
removeJobsByUserId(demoUser.id);
}

View File

@@ -1,10 +1,12 @@
import cron from 'node-cron';
import { config, inDevMode } from '../../utils.js';
import { inDevMode } from '../../utils.js';
import { trackMainEvent } from '../tracking/Tracker.js';
import { getSettings } from '../storage/settingsStorage.js';
async function runTask() {
const settings = await getSettings();
//make sure to only send tracking events if the user gave us the green light and we are not in dev mode
if (config.analyticsEnabled && !inDevMode()) {
if (settings.analyticsEnabled && !inDevMode()) {
await trackMainEvent();
}
}

View File

@@ -0,0 +1,274 @@
import { DEFAULT_HEADER } from './utils.js';
// Helper to safely coerce numbers
const toInt = (v, d) => {
const n = parseInt(v, 10);
return Number.isFinite(n) ? n : d;
};
/**
* Compute pre-launch configuration and flags for Puppeteer with bot prevention in mind.
* Returns language, user agent, viewport (with optional jitter), and additional launch args.
*
* @param {string} url
* @param {object} [options]
*/
export function getPreLaunchConfig(url, options = {}) {
const { hostname } = new URL(url);
const acceptLanguage = options.acceptLanguage || 'de-DE,de;q=0.9,en-US;q=0.7,en;q=0.5';
const langForFlag = acceptLanguage.split(',')[0];
const baseViewport = { width: 1366, height: 768, deviceScaleFactor: 1 };
const jitter = options.viewportJitter !== false ? Math.floor(Math.random() * 6) : 0; // 0..5 px
const width = toInt(options?.viewport?.width, baseViewport.width) + jitter;
const height = toInt(options?.viewport?.height, baseViewport.height) + jitter;
const deviceScaleFactor = toInt(options?.viewport?.deviceScaleFactor, baseViewport.deviceScaleFactor);
const viewport = { width, height, deviceScaleFactor };
const userAgent =
options.userAgent ||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
const windowSizeArg = `--window-size=${viewport.width},${viewport.height}`;
const langArg = `--lang=${langForFlag}`;
const extraArgs = [
'--disable-blink-features=AutomationControlled',
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
'--webrtc-ip-handling-policy=default_public_interface_only',
'--proxy-bypass-list=<-loopback>',
];
const headers = {
...DEFAULT_HEADER,
'Accept-Language': acceptLanguage,
'User-Agent': userAgent,
Referer: options?.referer || `https://${hostname}/`,
Connection: 'keep-alive',
DNT: '1',
};
const timezone = options?.timezone || 'Europe/Berlin';
return {
acceptLanguage,
langForFlag,
userAgent,
viewport,
windowSizeArg,
langArg,
extraArgs,
headers,
timezone,
humanDelay: options?.humanDelay !== false,
};
}
/**
* Apply bot-prevention hardening to a Puppeteer page.
* Sets UA, viewport, JS enabled, headers, timezone and injects stealth-like patches.
*
* @param {import('puppeteer').Page} page
* @param {ReturnType<typeof getPreLaunchConfig>} cfg
*/
export async function applyBotPreventionToPage(page, cfg) {
await page.setUserAgent(cfg.userAgent);
await page.setViewport(cfg.viewport);
await page.setJavaScriptEnabled(true);
await page.setExtraHTTPHeaders(cfg.headers);
try {
if (cfg.timezone) await page.emulateTimezone(cfg.timezone);
} catch {
// ignore timezone failures
}
// Inject patches as early as possible
await page.evaluateOnNewDocument(() => {
try {
// webdriver
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
// chrome runtime
// @ts-ignore
if (!window.chrome) {
// @ts-ignore
window.chrome = { runtime: {} };
}
// languages
// @ts-ignore
Object.defineProperty(navigator, 'languages', {
get: () => (window.localStorage.getItem('__LANGS__') || 'de-DE,de').split(','),
});
// plugins
// @ts-ignore
Object.defineProperty(navigator, 'plugins', {
get: () => [{}, {}, {}],
});
// platform and concurrency hints
// @ts-ignore
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
// @ts-ignore
if (typeof navigator.hardwareConcurrency === 'number' && navigator.hardwareConcurrency < 2) {
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 4 });
}
// @ts-ignore
if (typeof navigator.deviceMemory === 'number' && navigator.deviceMemory < 2) {
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
}
// userAgentData (Client Hints)
try {
// @ts-ignore
if ('userAgentData' in navigator) {
// @ts-ignore
Object.defineProperty(navigator, 'userAgentData', {
get: () => ({
brands: [
{ brand: 'Chromium', version: '126' },
{ brand: 'Google Chrome', version: '126' },
],
mobile: false,
platform: 'Windows',
getHighEntropyValues: async (hints) => {
const values = {
platform: 'Windows',
platformVersion: '15.0.0',
architecture: 'x86',
model: '',
uaFullVersion: '126.0.0.0',
bitness: '64',
};
const out = {};
for (const k of hints || []) if (k in values) out[k] = values[k];
return out;
},
}),
});
}
} catch {
//noop
}
// Permissions API
const origQuery = navigator.permissions && navigator.permissions.query;
if (origQuery) {
// @ts-ignore
navigator.permissions.query = (parameters) =>
origQuery.call(navigator.permissions, parameters).then((result) => {
if (parameters && parameters.name === 'notifications') {
Object.defineProperty(result, 'state', { get: () => Notification.permission });
}
return result;
});
}
// WebGL vendor/renderer
const patchWebGL = (proto) => {
if (!proto || !proto.getParameter) return;
const getParameter = proto.getParameter;
// @ts-ignore
proto.getParameter = function (param) {
const UNMASKED_VENDOR_WEBGL = 0x9245;
const UNMASKED_RENDERER_WEBGL = 0x9246;
if (param === UNMASKED_VENDOR_WEBGL) return 'Google Inc.';
if (param === UNMASKED_RENDERER_WEBGL)
return 'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 Ti Direct3D11 vs_5_0 ps_5_0)';
return getParameter.call(this, param);
};
};
// @ts-ignore
patchWebGL(WebGLRenderingContext?.prototype);
// @ts-ignore
patchWebGL(WebGL2RenderingContext?.prototype);
// AudioContext timestamp rounding consistency
const patchAudio = (Ctx) => {
try {
if (!Ctx) return;
const proto = Ctx.prototype;
const createOsc = proto.createOscillator;
proto.createOscillator = function () {
const osc = createOsc.call(this);
const start = osc.start;
osc.start = function (when) {
return start.call(this, when || 0);
};
return osc;
};
} catch {
//noop
}
};
// @ts-ignore
patchAudio(window.AudioContext);
// @ts-ignore
patchAudio(window.OfflineAudioContext);
// Navigator.connection
try {
// @ts-ignore
Object.defineProperty(navigator, 'connection', { get: () => undefined });
} catch {
//noop
}
// Consistent outer sizes
try {
const calcOuter = () => {
const w = window.innerWidth + 16;
const h = window.innerHeight + 88;
return { w, h };
};
const { w: outerW, h: outerH } = calcOuter();
// @ts-ignore
Object.defineProperty(window, 'outerWidth', { get: () => outerW });
// @ts-ignore
Object.defineProperty(window, 'outerHeight', { get: () => outerH });
} catch {
//noop
}
} catch {
//noop
}
});
}
/**
* Persist languages value before navigation via localStorage.
* @param {import('puppeteer').Page} page
* @param {ReturnType<typeof getPreLaunchConfig>} cfg
*/
export async function applyLanguagePersistence(page, cfg) {
await page.evaluateOnNewDocument((langs) => {
try {
window.localStorage.setItem('__LANGS__', langs);
} catch {
// noop
}
}, cfg.acceptLanguage.split(';')[0]);
}
/**
* Perform subtle human-like interactions post navigation.
* @param {import('puppeteer').Page} page
* @param {ReturnType<typeof getPreLaunchConfig>} cfg
*/
export async function applyPostNavigationHumanSignals(page, cfg) {
if (!cfg.humanDelay) return;
const delay = 200 + Math.floor(Math.random() * 400);
await new Promise((res) => setTimeout(res, delay));
try {
const vw = cfg.viewport.width;
const vh = cfg.viewport.height;
const mx = Math.floor(vw * (0.3 + Math.random() * 0.4));
const my = Math.floor(vh * (0.3 + Math.random() * 0.4));
await page.mouse.move(mx, my, { steps: 10 + Math.floor(Math.random() * 10) });
await page.mouse.wheel({ deltaY: 100 + Math.floor(Math.random() * 200) });
} catch {
// ignore if mouse is unavailable
}
}

View File

@@ -1,6 +1,12 @@
import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
import { debug, botDetected } from './utils.js';
import {
getPreLaunchConfig,
applyBotPreventionToPage,
applyLanguagePersistence,
applyPostNavigationHumanSignals,
} from './botPrevention.js';
import logger from '../logger.js';
import fs from 'fs';
import os from 'os';
@@ -27,23 +33,50 @@ export default async function execute(url, waitForSelector, options) {
removeUserDataDir = true;
}
const launchArgs = [
'--no-sandbox',
'--disable-gpu',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-crash-reporter',
'--no-first-run',
'--no-default-browser-check',
];
if (options?.proxyUrl) {
launchArgs.push(`--proxy-server=${options.proxyUrl}`);
}
// Prepare bot prevention pre-launch config
const preCfg = getPreLaunchConfig(url, options || {});
launchArgs.push(preCfg.langArg);
launchArgs.push(preCfg.windowSizeArg);
launchArgs.push(...preCfg.extraArgs);
browser = await puppeteer.launch({
headless: options.puppeteerHeadless ?? true,
args: [
'--no-sandbox',
'--disable-gpu',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-crash-reporter',
],
timeout: options.puppeteerTimeout || 30_000,
headless: options?.puppeteerHeadless ?? true,
args: launchArgs,
timeout: options?.puppeteerTimeout || 30_000,
userDataDir,
executablePath: options?.executablePath, // allow using system Chrome
});
page = await browser.newPage();
await page.setExtraHTTPHeaders(DEFAULT_HEADER);
await applyBotPreventionToPage(page, preCfg);
// Provide languages value before navigation
await applyLanguagePersistence(page, preCfg);
// Optional cookies
if (Array.isArray(options?.cookies) && options.cookies.length > 0) {
await page.setCookie(...options.cookies);
}
// Navigation
const response = await page.goto(url, {
waitUntil: 'domcontentloaded',
waitUntil: options?.waitUntil || 'domcontentloaded',
});
// Optionally wait and add subtle human-like interactions
await applyPostNavigationHumanSignals(page, preCfg);
let pageSource;
// if we're extracting data from a SPA, we must wait for the selector
if (waitForSelector != null) {
@@ -57,7 +90,7 @@ export default async function execute(url, waitForSelector, options) {
pageSource = await page.content();
}
const statusCode = response.status();
const statusCode = response?.status?.() ?? 200;
if (botDetected(pageSource, statusCode)) {
logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
@@ -66,7 +99,7 @@ export default async function execute(url, waitForSelector, options) {
result = pageSource || (await page.content());
}
} catch (error) {
logger.error('Error executing with puppeteer executor', error);
logger.warn('Error executing with puppeteer executor', error);
result = null;
} finally {
try {

View File

@@ -1,6 +1,4 @@
import markdown$0 from 'markdown';
import fs from 'fs';
const markdown = markdown$0.markdown;
export function markdown2Html(filePath) {
return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
return fs.readFileSync(filePath, 'utf8');
}

View File

@@ -1,116 +1,94 @@
import crypto from 'crypto';
const retention = 60 * 60 * 1000;
/**
* Internal cache storage.
* Maps a SHA-256 hash (string) to its expiry timestamp (number in ms).
* @type {Map<string, number>}
*/
const entries = new Map();
/**
* Reference to the currently scheduled cleanup timer.
* @type {NodeJS.Timeout | null}
*/
let timer = null;
/**
* Generate a SHA-256 hash from a list of input strings.
* Null or undefined values are ignored.
* Similarity cache
*
* @param {...(string|null|undefined)} strings - Input values to hash
* Maintains an in-memory Set of content hashes to detect whether a listing
* (identified by a tuple of title, price and address) has been seen before.
*
* Design notes:
* - The cache is refreshed periodically from persistent storage. To avoid
* modification-during-iteration issues, the refresh builds a new Set and
* atomically swaps the reference instead of mutating in place.
* - Hashing ignores null/undefined values but preserves falsy-yet-valid values
* like 0. Non-string values are coerced to strings before hashing.
*
* This module has no persistence of its own; it relies on
* getAllEntriesFromListings() for data hydration.
* @module similarityCache
*/
import crypto from 'crypto';
import { getAllEntriesFromListings } from '../storage/listingsStorage.js';
/** @type {number} Refresh interval in milliseconds (defaults to one hour). */
const reloadCycle = 60 * 60 * 1000; // every hour, refresh
/**
* Internal cache of content hashes for known listings.
*
* Each entry is an SHA-256 hex digest produced by toHash(title, price, address).
* @type {Set<string>}
*/
let cache = new Set();
export const startSimilarityCacheReloader = () => {
// Periodically refresh the cache from storage
setInterval(() => {
initSimilarityCache();
}, reloadCycle);
};
/**
* Initialize or refresh the similarity cache from persistent storage.
*
* Reads all stored listings via getAllEntriesFromListings(), computes a hash for
* each, and swaps the in-memory Set atomically to avoid in-place mutations that
* could interfere with concurrent iteration.
*
* This function is idempotent and safe to call at any time.
* @returns {void}
*/
export const initSimilarityCache = () => {
const allEntries = getAllEntriesFromListings();
const newCache = new Set();
for (const entry of allEntries) {
newCache.add(toHash(entry?.title, entry?.price, entry?.address));
}
// Atomic swap to avoid mutating the cache while it may be iterated elsewhere
cache = newCache;
};
/**
* Check if a listing is already known and add it to the cache if not.
*
* The listing is identified by the combination of its title, price and
* address. Null/undefined fields are ignored during hashing. Falsy-but-valid
* values (e.g., price 0) are preserved.
*
* @param {Object} params - Listing fields
* @param {string|undefined|null} params.title - The listing title
* @param {string|undefined|null} params.address - The listing address
* @param {number|string|undefined|null} params.price - The listing price
* @returns {boolean} true if the entry already existed in the cache (duplicate), otherwise false
*/
export const checkAndAddEntry = ({ title, address, price }) => {
const hash = toHash(title, price, address);
if (cache.has(hash)) {
return true;
}
cache.add(hash);
return false;
};
/**
* Generate an SHA-256 hash from a list of input values.
* Null or undefined values are ignored. Falsy but valid values like 0 are preserved.
* Non-string values are coerced to strings prior to hashing.
*
* @param {...(string|number|null|undefined)} strings - Input values to hash
* @returns {string} Hexadecimal hash
*/
function toHash(...strings) {
return crypto.createHash('sha256').update(strings.filter(Boolean).join('|')).digest('hex');
}
/**
* Cleanup expired cache entries and schedule the next cleanup run.
* This function is invoked automatically by scheduled timers.
*
* @private
*/
function runCleanup() {
const now = Date.now();
for (const [hash, expiry] of entries) {
if (expiry <= now) entries.delete(hash);
}
scheduleNext();
}
/**
* Find the soonest expiry timestamp among all cache entries
* and schedule a one-shot timer that will trigger at that time.
* Cancels any existing timer before scheduling a new one.
*
* @private
*/
function scheduleNext() {
if (timer) {
clearTimeout(timer);
timer = null;
}
let next = Infinity;
const now = Date.now();
for (const expiry of entries.values()) {
if (expiry > now && expiry < next) next = expiry;
}
if (next !== Infinity) {
timer = setTimeout(runCleanup, Math.max(0, next - now));
}
}
/**
* Add or refresh a cache entry for the given title and address.
* The entry will automatically expire after the configured retention window.
*
* @param {string} title - The title used to build the cache key
* @param {string} address - The address used to build the cache key
*/
export function addCacheEntry(title, address) {
const hash = toHash(title, address);
const expiry = Date.now() + retention;
entries.set(hash, expiry);
scheduleNext();
}
/**
* Check if a cache entry with the same title and address exists
* and is still valid (not expired).
*
* @param {string} title - The title used to build the cache key
* @param {string} address - The address used to build the cache key
* @returns {boolean} True if a valid cache entry exists, false otherwise
*/
export function hasSimilarEntries(title, address) {
const hash = toHash(title, address);
const expiry = entries.get(hash);
if (expiry == null) return false;
if (expiry <= Date.now()) {
entries.delete(hash);
scheduleNext();
return false;
}
return true;
}
/**
* Stop any scheduled cleanup timers and prevent further automatic cleanup.
* Entries that are already in the cache will remain until removed manually
* or until cleanup is started again by adding new entries.
*/
export function stopCacheCleanup() {
if (timer) clearTimeout(timer);
timer = null;
}
/**
* this is only for test purposes
*/
export function invalidateAllForTest() {
for (const key of entries.keys()) {
entries.set(key, 0);
}
runCleanup();
const normalized = strings
.filter((v) => v !== null && v !== undefined)
.map((v) => (typeof v === 'string' ? v : String(v)));
return crypto.createHash('sha256').update(normalized.join('|')).digest('hex');
}

View File

@@ -2,7 +2,7 @@ import fs from 'fs';
import path from 'path';
import Database from 'better-sqlite3';
import logger from '../../services/logger.js';
import { config } from '../../utils.js';
import { readConfigFromStorage } from '../../utils.js';
/**
* SqliteConnection
@@ -25,6 +25,15 @@ import { config } from '../../utils.js';
class SqliteConnection {
static #db = null;
static #sqlLiteCfg = null;
static async init() {
if (this.#sqlLiteCfg == null) {
readConfigFromStorage().then((c) => {
this.#sqlLiteCfg = c.sqlitepath;
});
}
}
/**
* Returns a singleton instance of better-sqlite3 Database.
* Respects env var SQLITE_DB_PATH and defaults to db/listings.db.
@@ -32,9 +41,12 @@ class SqliteConnection {
static getConnection() {
if (this.#db) return this.#db;
if (this.#sqlLiteCfg == null) {
logger.warn('No sqlitepath configured. Using default db/listings.db');
}
// Interpret config.sqlitepath as a directory relative to project root when it starts with '/'
const cfg = typeof config === 'object' && config ? config.sqlitepath : undefined;
const rawDir = cfg && cfg.length > 0 ? cfg : '/db';
const rawDir = this.#sqlLiteCfg && this.#sqlLiteCfg.length > 0 ? this.#sqlLiteCfg : '/db';
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
const dbPath = path.join(absDir, 'listings.db');

View File

@@ -16,7 +16,16 @@ import { toJson, fromJson } from '../../utils.js';
* @param {string} params.userId - Owner user id for inserts; preserved on updates.
* @returns {void}
*/
export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
export const upsertJob = ({
jobId,
name,
blacklist = [],
enabled = true,
provider,
notificationAdapter,
userId,
shareWithUsers = [],
}) => {
const id = jobId || nanoid();
const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0];
const ownerId = existing ? existing.user_id : userId;
@@ -27,21 +36,23 @@ export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provide
name = @name,
blacklist = @blacklist,
provider = @provider,
notification_adapter = @notification_adapter
notification_adapter = @notification_adapter,
shared_with_user = @shareWithUsers
WHERE id = @id`,
{
id,
enabled: enabled ? 1 : 0,
name: name ?? null,
blacklist: toJson(blacklist ?? []),
shareWithUsers: toJson(shareWithUsers ?? []),
provider: toJson(provider ?? []),
notification_adapter: toJson(notificationAdapter ?? []),
},
);
} else {
SqliteConnection.execute(
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter)
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter)`,
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user)
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers)`,
{
id,
user_id: ownerId,
@@ -49,6 +60,7 @@ export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provide
name: name ?? null,
blacklist: toJson(blacklist ?? []),
provider: toJson(provider ?? []),
shareWithUsers: toJson(shareWithUsers ?? []),
notification_adapter: toJson(notificationAdapter ?? []),
},
);
@@ -129,6 +141,7 @@ export const getJobs = () => {
j.name,
j.blacklist,
j.provider,
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
FROM jobs j
@@ -139,6 +152,7 @@ export const getJobs = () => {
enabled: !!row.enabled,
blacklist: fromJson(row.blacklist, []),
provider: fromJson(row.provider, []),
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
}));
};

View File

@@ -152,8 +152,9 @@ export const storeListings = (jobId, providerId, listings) => {
*/
function extractNumber(str) {
if (!str) return null;
const match = str.replace(/[.,]/g, '').match(/\d+/);
return match ? +match[0] : null;
const cleaned = str.replace(/\./g, '').replace(',', '.');
const num = parseFloat(cleaned);
return isNaN(num) ? null : num;
}
/**
@@ -213,7 +214,10 @@ export const queryListings = ({
params.userId = userId || '__NO_USER__';
// user scoping (non-admin only): restrict to listings whose job belongs to user
if (!isAdmin) {
whereParts.push(`(j.user_id = @userId)`);
// Include listings from jobs owned by the user or jobs shared with the user
whereParts.push(
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
);
}
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
params.filter = `%${String(freeTextFilter).trim()}%`;
@@ -307,8 +311,8 @@ export const deleteListingsByJobId = (jobId) => {
if (!jobId) return;
return SqliteConnection.execute(
`DELETE
FROM listings
WHERE job_id = @jobId`,
FROM listings
WHERE job_id = @jobId`,
{ jobId },
);
};
@@ -329,3 +333,13 @@ export const deleteListingsById = (ids) => {
ids,
);
};
/**
* Return all listings with only the fields: title, address, and price.
* This is the single helper requested for simple consumers.
*
* @returns {{title: string|null, address: string|null, price: number|null}[]}
*/
export const getAllEntriesFromListings = () => {
return SqliteConnection.query(`SELECT title, address, price FROM listings`);
};

View File

@@ -0,0 +1,7 @@
// Migration: Adding a new table to store if somebody shared a job with someone
export function up(db) {
db.exec(`
ALTER TABLE jobs ADD COLUMN shared_with_user jsonb DEFAULT '[]'
`);
}

View File

@@ -0,0 +1,73 @@
// Migration: Adding a settings table to store important (config) settings instead of using config file
import fs from 'fs';
import path from 'path';
import { nanoid } from 'nanoid';
import logger from '../../../logger.js';
export function up(db) {
db.exec(`
CREATE TABLE IF NOT EXISTS settings
(
id TEXT PRIMARY KEY,
create_date INTEGER NOT NULL,
user_id TEXT,
name TEXT NOT NULL,
value jsonb NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_settings_name ON settings (name);
`);
// Helper to insert one setting row
const insertSetting = (name, rawValue) => {
try {
const id = nanoid();
const createDate = Date.now();
const value = JSON.stringify(rawValue);
db.prepare(
`INSERT INTO settings (id, create_date, name, value)
VALUES (@id, @create_date, @name, @value)`,
).run({ id, create_date: createDate, name, value });
} catch {
// Ignore duplicate inserts if any (unique by name)
}
};
// Migrate currently existing config.json into settings
try {
const configPath = path.resolve(process.cwd(), 'conf', 'config.json');
// Defaults
const defaults = {
interval: '60',
port: 9998,
workingHours: { from: '', to: '' },
demoMode: false,
analyticsEnabled: true,
};
let config = {};
if (fs.existsSync(configPath)) {
const file = fs.readFileSync(configPath, 'utf8');
try {
config = JSON.parse(file) || {};
} catch (parseErr) {
// If parsing fails, still proceed with defaults
logger.error(parseErr);
config = {};
}
}
// Insert each known setting, using the value from config when present, otherwise default
insertSetting('interval', config.interval != null ? config.interval : defaults.interval);
insertSetting('port', config.port != null ? config.port : defaults.port);
insertSetting('workingHours', config.workingHours != null ? config.workingHours : defaults.workingHours);
insertSetting('demoMode', config.demoMode != null ? config.demoMode : defaults.demoMode);
insertSetting(
'analyticsEnabled',
config.analyticsEnabled != null ? config.analyticsEnabled : defaults.analyticsEnabled,
);
} catch (e) {
logger.error(e);
}
}

View File

@@ -0,0 +1,87 @@
import { nanoid } from 'nanoid';
import SqliteConnection from './SqliteConnection.js';
import { fromJson, readConfigFromStorage, toJson } from '../../utils.js';
// In-memory cache for compiled settings config
/** @type {Record<string, any>|null} */
let cachedSettingsConfig = null;
/**
* Build a config object from DB rows of settings.
* - Unwraps stored shape { value: any } into raw values.
* - Add additional config values from file config. E.g. sqlite part cannot be stored in db for obvious reasons ;)
* @param {{name:string, value:string|null}[]} rows
* @param {{name:value}} configValues
* @returns {Record<string, any>}
*/
function compileSettings(rows, configValues) {
const config = {};
for (const r of rows) {
const parsed = fromJson(r.value, null);
// unwrap { value: any } if present
config[r.name] = parsed && typeof parsed === 'object' && 'value' in parsed ? parsed.value : parsed;
}
return {
...config,
...configValues,
};
}
/**
* Force reload the settings config cache from DB and return it.
* @returns {Record<string, any>}
*/
export async function refreshSettingsCache() {
const rows = SqliteConnection.query(`SELECT name, value FROM settings`);
const configValues = await readConfigFromStorage();
cachedSettingsConfig = compileSettings(rows, configValues);
return cachedSettingsConfig;
}
/**
* Get the compiled settings config. Loads it once and caches the result.
* @returns {Record<string, any>}
*/
export async function getSettings() {
if (cachedSettingsConfig == null) {
return refreshSettingsCache();
}
return cachedSettingsConfig;
}
/**
* Upsert settings rows.
* - Accepts an object map of name -> value, or an entry {name, value}.
* - id: random string (nanoid) when inserting
* - create_date: epoch ms when inserting
* - name: unique key
* - value: JSON string of the raw value (no wrapper)
* @param {Record<string, any>|{name:string, value:any}|[string, any][]} settingsMapOrEntry
* @returns {void}
*/
// Upsert one or more settings by name. Accepts either a single pair or an object map.
// Preferred usage: upsertSettings({ settingName: any, another: any })
export function upsertSettings(settingsMapOrEntry, userId = null) {
const entries = Array.isArray(settingsMapOrEntry)
? settingsMapOrEntry
: typeof settingsMapOrEntry === 'object' &&
settingsMapOrEntry != null &&
'name' in settingsMapOrEntry &&
'value' in settingsMapOrEntry
? [[settingsMapOrEntry.name, settingsMapOrEntry.value]]
: Object.entries(settingsMapOrEntry || {});
for (const [name, rawValue] of entries) {
const id = nanoid();
const create_date = Date.now();
const json = toJson(rawValue);
SqliteConnection.execute(
`INSERT INTO settings (id, create_date, name, value, user_id)
VALUES (@id, @create_date, @name, @value, @userId)
ON CONFLICT(name) DO UPDATE SET value = excluded.value`,
{ id, create_date, name, value: json, userId },
);
}
// keep cache in sync
refreshSettingsCache();
}

View File

@@ -1,7 +1,7 @@
import { config } from '../../utils.js';
import * as hasher from '../security/hash.js';
import { nanoid } from 'nanoid';
import SqliteConnection from './SqliteConnection.js';
import { getSettings } from './settingsStorage.js';
/**
* Get all users.
@@ -129,8 +129,9 @@ export const removeUser = (userId) => {
* Security: The demo user's password is set to a known value ('demo') and should only be enabled in demoMode.
* @returns {void}
*/
export const ensureDemoUserExists = () => {
if (!config.demoMode) {
export const ensureDemoUserExists = async () => {
const settings = await getSettings();
if (!settings.demoMode) {
// Remove demo user (and cascade delete their jobs/listings)
SqliteConnection.execute(`DELETE FROM users WHERE username = 'demo'`);
return;

View File

@@ -1,9 +1,10 @@
import { getJobs } from '../storage/jobStorage.js';
import { getUniqueId } from './uniqueId.js';
import { config, getPackageVersion, inDevMode } from '../../utils.js';
import { getPackageVersion, inDevMode } from '../../utils.js';
import os from 'os';
import fetch from 'node-fetch';
import logger from '../logger.js';
import { getSettings } from '../storage/settingsStorage.js';
const deviceId = getUniqueId() || 'N/A';
const version = await getPackageVersion();
@@ -11,7 +12,8 @@ const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
export const trackMainEvent = async () => {
try {
if (config.analyticsEnabled && !inDevMode()) {
const settings = await getSettings();
if (settings.analyticsEnabled && !inDevMode()) {
const activeProvider = new Set();
const activeAdapter = new Set();
@@ -44,7 +46,8 @@ export const trackMainEvent = async () => {
* Note, this will only be used when Fredy runs in demo mode
*/
export async function trackDemoAccessed() {
if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
const settings = await getSettings();
if (settings.analyticsEnabled && !inDevMode() && settings.demoMode) {
try {
await fetch(`${FREDY_TRACKING_URL}/demo/accessed`, {
method: 'POST',
@@ -56,7 +59,8 @@ export async function trackDemoAccessed() {
}
}
function enrichTrackingObject(trackingObject) {
async function enrichTrackingObject(trackingObject) {
const settings = await getSettings();
const operatingSystem = os.platform();
const osVersion = os.release();
const arch = process.arch;
@@ -65,7 +69,7 @@ function enrichTrackingObject(trackingObject) {
return {
...trackingObject,
isDemo: config.demoMode,
isDemo: settings.demoMode,
operatingSystem,
osVersion,
arch,

View File

@@ -215,10 +215,6 @@ export async function refreshConfig() {
try {
config = await readConfigFromStorage();
//backwards compatibility...
config.analyticsEnabled ??= null;
config.demoMode ??= false;
// default sqlitepath when missing in older configs
config.sqlitepath ??= '/db';
} catch (error) {
config = { ...DEFAULT_CONFIG };
@@ -306,7 +302,6 @@ export {
getDirName,
sleep,
randomBetween,
config,
buildHash,
getPackageVersion,
toJson,

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "14.1.1",
"version": "15.0.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",
@@ -12,6 +12,7 @@
"format": "prettier --write \"**/*.js\"",
"format:check": "prettier --check \"**/*.js\"",
"test": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 test/**/*.test.js",
"testGH": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 --exclude test/provider/immonet.test.js --exclude test/provider/immowelt.test.js test/**/*.test.js",
"lint": "eslint .",
"lint:fix": "yarn lint --fix",
"migratedb": "node lib/services/storage/migrations/migrate.js",
@@ -56,58 +57,57 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-icons": "^2.86.0",
"@douyinfe/semi-ui": "2.86.0",
"@douyinfe/semi-icons": "^2.89.0",
"@douyinfe/semi-ui": "2.89.0",
"@sendgrid/mail": "8.1.6",
"@visactor/react-vchart": "^2.0.5",
"@visactor/vchart": "^2.0.5",
"@visactor/react-vchart": "^2.0.10",
"@visactor/vchart": "^2.0.10",
"@visactor/vchart-semi-theme": "^1.12.2",
"@vitejs/plugin-react": "5.0.4",
"better-sqlite3": "^12.4.1",
"body-parser": "2.2.0",
"@vitejs/plugin-react": "5.1.2",
"better-sqlite3": "^12.5.0",
"body-parser": "2.2.1",
"cheerio": "^1.1.2",
"cookie-session": "2.1.1",
"handlebars": "4.7.8",
"lodash": "4.17.21",
"markdown": "^0.5.0",
"nanoid": "5.1.6",
"node-cron": "^4.2.1",
"node-fetch": "3.3.2",
"node-mailjet": "6.0.9",
"p-throttle": "^8.0.0",
"node-mailjet": "6.0.11",
"p-throttle": "^8.1.0",
"package-up": "^5.0.0",
"puppeteer": "^24.23.0",
"puppeteer": "^24.32.1",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.3.1",
"react": "18.3.1",
"react-dom": "18.3.1",
"react-router": "7.9.3",
"react-router-dom": "7.9.3",
"react-router": "7.10.1",
"react-router-dom": "7.10.1",
"restana": "5.1.0",
"semver": "^7.7.2",
"semver": "^7.7.3",
"serve-static": "2.2.0",
"slack": "11.0.2",
"vite": "7.1.9",
"vite": "7.2.7",
"x-var": "^3.0.1",
"zustand": "^5.0.8"
"zustand": "^5.0.9"
},
"devDependencies": {
"@babel/core": "7.28.4",
"@babel/eslint-parser": "7.28.4",
"@babel/preset-env": "7.28.3",
"@babel/preset-react": "7.27.1",
"chai": "6.2.0",
"eslint": "9.36.0",
"@babel/core": "7.28.5",
"@babel/eslint-parser": "7.28.5",
"@babel/preset-env": "7.28.5",
"@babel/preset-react": "7.28.5",
"chai": "6.2.1",
"eslint": "9.39.1",
"eslint-config-prettier": "10.1.8",
"eslint-plugin-react": "7.37.5",
"esmock": "2.7.3",
"history": "5.3.0",
"husky": "9.1.7",
"less": "4.4.1",
"lint-staged": "16.2.3",
"mocha": "11.7.4",
"nodemon": "^3.1.10",
"prettier": "3.6.2"
"less": "4.4.2",
"lint-staged": "16.2.7",
"mocha": "11.7.5",
"nodemon": "^3.1.11",
"prettier": "3.7.4"
}
}

View File

@@ -1,53 +0,0 @@
import { expect } from 'chai';
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { mockFredy } from '../utils.js';
describe('FredyRuntime', () => {
afterEach(() => {
similarityCache.invalidateAllForTest();
});
after(() => {
similarityCache.stopCacheCleanup();
});
describe('_filterBySimilarListings', () => {
let fredyRuntime;
beforeEach(async () => {
const FredyRuntime = await mockFredy();
fredyRuntime = new FredyRuntime({}, null, 'dummy-provider', 'dummy-job', similarityCache);
});
it('should filter out listings with similar title and address already in cache', () => {
similarityCache.addCacheEntry('Penthouse', 'Mustermann Straße 1');
const listings = [
{ id: '1', title: 'Penthouse', address: 'Mustermann Straße 1' },
{ id: '2', title: 'Nice apartment', address: 'Mustermann Straße 15' },
];
const result = fredyRuntime._filterBySimilarListings(listings);
expect(result).to.have.length(1);
expect(result[0].id).to.equal('2');
expect(result[0].title).to.equal('Nice apartment');
expect(similarityCache.hasSimilarEntries('Nice apartment', 'Mustermann Straße 15')).to.be.true;
});
it('should handle listings with null or undefined address', () => {
const listings = [
{ id: '1', title: 'Penthouse', address: null },
{ id: '2', title: 'Nice apartment', address: undefined },
];
const result = fredyRuntime._filterBySimilarListings(listings);
expect(result).to.have.length(2);
expect(similarityCache.hasSimilarEntries('Penthouse', null)).to.be.true;
expect(similarityCache.hasSimilarEntries('Nice apartment', undefined)).to.be.true;
});
});
});

View File

@@ -5,9 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/einsAImmobilien.js';
describe('#einsAImmobilien testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.einsAImmobilien, [], []);
it('should test einsAImmobilien provider', async () => {
const Fredy = await mockFredy();

View File

@@ -5,9 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/immobilienDe.js';
describe('#immobilien.de testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immobilienDe, [], []);
it('should test immobilien.de provider', async () => {
const Fredy = await mockFredy();

View File

@@ -5,10 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/immonet.js';
describe('#immonet testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test immonet provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.immonet, [], []);

View File

@@ -5,10 +5,6 @@ import { get } from '../mocks/mockNotification.js';
import * as provider from '../../lib/provider/immoscout.js';
describe('#immoscout provider testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immoscout, [], []);
it('should test immoscout provider', async () => {
const Fredy = await mockFredy();

View File

@@ -5,9 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/immoswp.js';
describe('#immoswp testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immoswp, [], []);
it('should test immoswp provider', async () => {
const Fredy = await mockFredy();

View File

@@ -5,10 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/immowelt.js';
describe('#immowelt testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test immowelt provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.immowelt, [], []);

View File

@@ -5,9 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/kleinanzeigen.js';
describe('#kleinanzeigen testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test kleinanzeigen provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.kleinanzeigen, [], []);

View File

@@ -5,10 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/mcMakler.js';
describe('#mcMakler testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test mcMakler provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.mcMakler, []);

View File

@@ -5,9 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/neubauKompass.js';
describe('#neubauKompass testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.neubauKompass, [], []);
it('should test neubauKompass provider', async () => {
const Fredy = await mockFredy();

View File

@@ -0,0 +1,33 @@
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import { expect } from 'chai';
import * as provider from '../../lib/provider/ohneMakler.js';
describe('#ohneMakler testsuite()', () => {
it('should test ohneMakler provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.ohneMakler, []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'ohneMakler', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('ohneMakler');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.address).to.be.not.empty;
});
});
});

View File

@@ -5,10 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/regionalimmobilien24.js';
describe('#regionalimmobilien24 testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test regionalimmobilien24 provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.regionalimmobilien24, []);

View File

@@ -5,10 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/sparkasse.js';
describe('#sparkasse testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test sparkasse provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.sparkasse, []);

View File

@@ -32,6 +32,10 @@
"url": "https://www.mcmakler.de/immobilien/results?placeId=62649&search=Leipzig%252C+Sachsen&propertyTypes=APARTMENT&page=0",
"enabled": true
},
"ohneMakler": {
"url": "https://www.ohne-makler.net/immobilien/wohnung-kaufen/nordrhein-westfalen/dusseldorf/",
"enabled": true
},
"neubauKompass": {
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
"enabled": true

View File

@@ -5,9 +5,6 @@ import { expect } from 'chai';
import * as provider from '../../lib/provider/wgGesucht.js';
describe('#wgGesucht testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.wgGesucht, [], []);
it('should test wgGesucht provider', async () => {
const Fredy = await mockFredy();

View File

@@ -0,0 +1,99 @@
import { describe, it } from 'mocha';
import { expect } from 'chai';
import {
getPreLaunchConfig,
applyBotPreventionToPage,
applyLanguagePersistence,
applyPostNavigationHumanSignals,
} from '../../../lib/services/extractor/botPrevention.js';
describe('botPrevention helper', () => {
it('getPreLaunchConfig builds deterministic values when jitter disabled', () => {
const url = 'https://example.com/some/path';
const options = {
acceptLanguage: 'de-DE,de;q=0.9',
userAgent: 'TestAgent/1.0',
viewport: { width: 1200, height: 700, deviceScaleFactor: 2 },
viewportJitter: false,
referer: 'https://example.com/ref',
timezone: 'Europe/Berlin',
};
const cfg = getPreLaunchConfig(url, options);
expect(cfg.acceptLanguage).to.equal('de-DE,de;q=0.9');
expect(cfg.langArg).to.equal('--lang=de-DE');
expect(cfg.windowSizeArg).to.equal('--window-size=1200,700');
expect(cfg.viewport).to.deep.equal({ width: 1200, height: 700, deviceScaleFactor: 2 });
expect(cfg.userAgent).to.equal('TestAgent/1.0');
expect(cfg.headers['Accept-Language']).to.equal('de-DE,de;q=0.9');
expect(cfg.headers['User-Agent']).to.equal('TestAgent/1.0');
expect(cfg.headers.Referer).to.equal('https://example.com/ref');
expect(cfg.extraArgs).to.include('--disable-blink-features=AutomationControlled');
expect(cfg.extraArgs).to.include('--proxy-bypass-list=<-loopback>');
});
it('applyBotPreventionToPage sets UA, viewport, headers and injects patches', async () => {
const calls = [];
const page = {
setUserAgent: async (ua) => calls.push(['setUserAgent', ua]),
setViewport: async (vp) => calls.push(['setViewport', vp]),
setJavaScriptEnabled: async (on) => calls.push(['setJavaScriptEnabled', on]),
setExtraHTTPHeaders: async (h) => calls.push(['setExtraHTTPHeaders', h]),
emulateTimezone: async (tz) => calls.push(['emulateTimezone', tz]),
evaluateOnNewDocument: async (fn) => calls.push(['evaluateOnNewDocument', typeof fn]),
};
const cfg = getPreLaunchConfig('https://example.org/', {
userAgent: 'Foo/Bar',
acceptLanguage: 'en-US,en',
viewport: { width: 1000, height: 600, deviceScaleFactor: 1 },
viewportJitter: false,
timezone: 'UTC',
});
await applyBotPreventionToPage(page, cfg);
expect(calls[0]).to.deep.equal(['setUserAgent', 'Foo/Bar']);
expect(calls.some((c) => c[0] === 'setViewport' && c[1].width === 1000 && c[1].height === 600)).to.equal(true);
expect(calls.some((c) => c[0] === 'setJavaScriptEnabled' && c[1] === true)).to.equal(true);
const headerCall = calls.find((c) => c[0] === 'setExtraHTTPHeaders');
expect(headerCall).to.exist;
expect(headerCall[1]['Accept-Language']).to.equal('en-US,en');
expect(headerCall[1]['User-Agent']).to.equal('Foo/Bar');
expect(calls.some((c) => c[0] === 'emulateTimezone' && c[1] === 'UTC')).to.equal(true);
expect(calls.some((c) => c[0] === 'evaluateOnNewDocument' && c[1] === 'function')).to.equal(true);
});
it('applyLanguagePersistence stores languages early', async () => {
const calls = [];
const page = {
evaluateOnNewDocument: async (fn, arg) => calls.push(['evaluateOnNewDocument', typeof fn, arg]),
};
const cfg = getPreLaunchConfig('https://example.org/', {
acceptLanguage: 'de-DE,de;q=0.9',
viewportJitter: false,
});
await applyLanguagePersistence(page, cfg);
const call = calls[0];
expect(call[0]).to.equal('evaluateOnNewDocument');
expect(call[1]).to.equal('function');
expect(call[2]).to.equal('de-DE,de');
});
it('applyPostNavigationHumanSignals moves mouse and scrolls when enabled', async () => {
const mouseCalls = [];
const page = {
mouse: {
move: async (x, y, opts) => mouseCalls.push(['move', x, y, opts && typeof opts.steps === 'number']),
wheel: async (opts) => mouseCalls.push(['wheel', typeof opts.deltaY === 'number']),
},
};
const cfg = {
humanDelay: true,
viewport: { width: 1200, height: 800 },
};
await applyPostNavigationHumanSignals(page, cfg);
expect(mouseCalls.some((c) => c[0] === 'move')).to.equal(true);
expect(mouseCalls.some((c) => c[0] === 'wheel')).to.equal(true);
});
});

View File

@@ -1,30 +0,0 @@
import { expect } from 'chai';
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
describe('similarityCheck', () => {
it('should return true on duplicate', () => {
similarityCache.addCacheEntry('Hello World', 'Test');
expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.true;
});
it('should return true even if one value is null', () => {
similarityCache.addCacheEntry('Hello World', null);
expect(similarityCache.hasSimilarEntries('Hello World', null)).to.be.true;
});
it('should return true even if one value is an obj', () => {
similarityCache.addCacheEntry('Hello World', [{ TR: 'OLOLO' }]);
expect(similarityCache.hasSimilarEntries('Hello World', [{ TR: 'OLOLO' }])).to.be.true;
});
it('should return false when no duplicate', () => {
similarityCache.addCacheEntry('Hello World__', 'Test');
expect(similarityCache.hasSimilarEntries('Hello World___', 'Test')).to.be.false;
});
it('should return false when no duplicate', () => {
expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.true;
similarityCache.invalidateAllForTest();
expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.false;
});
});

View File

@@ -0,0 +1,62 @@
import { expect } from 'chai';
import esmock from 'esmock';
// Helper to create module under test with mocks
async function loadModuleWith({ entries = [] } = {}) {
const mod = await esmock('../../lib/services/similarity-check/similarityCache.js', {
// Mock the storage to return our controlled entries
'../../lib/services/storage/listingsStorage.js': {
getAllEntriesFromListings: () => entries,
},
});
return mod;
}
describe('similarityCache', () => {
it('initSimilarityCache builds cache from storage and enables duplicate detection', async () => {
const entries = [
{ title: 'A', price: 1000, address: 'Main 1' },
{ title: 'B', price: 0, address: 'Zero St' },
];
const { initSimilarityCache, checkAndAddEntry } = await loadModuleWith({ entries });
// Initially, duplicates should not be detected for new data
expect(checkAndAddEntry({ title: 'X', price: 200, address: 'Y' })).to.equal(false);
// Now initialize from storage
initSimilarityCache();
// Exact duplicates should be detected
expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).to.equal(true);
// Ensure falsy-but-valid price 0 is preserved by hashing and detected as duplicate
expect(checkAndAddEntry({ title: 'B', price: 0, address: 'Zero St' })).to.equal(true);
});
it('checkAndAddEntry returns false for new entry then true for duplicate on second call', async () => {
const { checkAndAddEntry } = await loadModuleWith();
const first = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' });
const second = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' });
expect(first).to.equal(false);
expect(second).to.equal(true);
});
it('hashing ignores null/undefined but preserves 0 via behavior', async () => {
const { checkAndAddEntry } = await loadModuleWith();
// Add baseline (null address ignored)
const add1 = checkAndAddEntry({ title: 'T', price: 1, address: null });
expect(add1).to.equal(false);
// Duplicate with undefined address should match
const dup = checkAndAddEntry({ title: 'T', price: 1, address: undefined });
expect(dup).to.equal(true);
// Now test that price 0 is preserved (not filtered out)
const addZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' });
expect(addZero).to.equal(false);
const dupZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' });
expect(dupZero).to.equal(true);
});
});

View File

@@ -6,7 +6,7 @@ import { send } from './mocks/mockNotification.js';
export const providerConfig = JSON.parse(await readFile(new URL('./provider/testProvider.json', import.meta.url)));
export const mockFredy = async () => {
return await esmock('../lib/FredyRuntime', {
return await esmock('../lib/FredyPipeline', {
'../lib/services/storage/listingsStorage.js': {
...mockStore,
},

View File

@@ -21,6 +21,7 @@ import Navigation from './components/navigation/Navigation.jsx';
import { Layout } from '@douyinfe/semi-ui';
import FredyFooter from './components/footer/FredyFooter.jsx';
import ProcessingTimes from './views/jobs/ProcessingTimes.jsx';
import WatchlistManagement from './views/listings/management/WatchlistManagement.jsx';
export default function FredyApp() {
const actions = useActions();
@@ -34,9 +35,11 @@ export default function FredyApp() {
async function init() {
await actions.user.getCurrentUser();
if (!needsLogin()) {
await actions.features.getFeatures();
await actions.provider.getProvider();
await actions.jobs.getJobs();
await actions.jobs.getProcessingTimes();
await actions.jobs.getSharableUserList();
await actions.notificationAdapter.getAdapter();
await actions.generalSettings.getGeneralSettings();
await actions.versionUpdate.getVersionUpdate();
@@ -90,6 +93,7 @@ export default function FredyApp() {
<Route path="/jobs/insights/:jobId" element={<JobInsight />} />
<Route path="/jobs" element={<Jobs />} />
<Route path="/listings" element={<Listings />} />
<Route path="/watchlistManagement" element={<WatchlistManagement />} />
{/* Permission-aware routes */}
<Route

View File

@@ -5,6 +5,8 @@
justify-content: space-between;
align-items: center;
height: 1.7rem;
border-radius: .3rem;
border-top: 1px solid #45464b;
&__version {
padding-left: .5rem;

View File

@@ -1,12 +1,13 @@
import React from 'react';
import { Nav } from '@douyinfe/semi-ui';
import { IconUser, IconStar, IconSetting, IconTerminal } from '@douyinfe/semi-icons';
import { IconStar, IconSetting, IconTerminal } from '@douyinfe/semi-icons';
import logoWhite from '../../assets/logo_white.png';
import Logout from '../logout/Logout.jsx';
import { useLocation, useNavigate } from 'react-router-dom';
import './Navigate.less';
import { useScreenWidth } from '../../hooks/screenWidth.js';
import { useFeature } from '../../hooks/featureHook.js';
export default function Navigation({ isAdmin }) {
const navigate = useNavigate();
@@ -14,15 +15,28 @@ export default function Navigation({ isAdmin }) {
const width = useScreenWidth();
const collapsed = width <= 850;
const watchlistFeature = useFeature('WATCHLIST_MANAGEMENT') || false;
const items = [
{ itemKey: '/jobs', text: 'Jobs', icon: <IconTerminal /> },
{ itemKey: '/listings', text: 'Found Listings', icon: <IconStar /> },
{ itemKey: '/listings', text: 'Listings', icon: <IconStar /> },
];
if (isAdmin) {
items.push({ itemKey: '/users', text: 'User Management', icon: <IconUser /> });
items.push({ itemKey: '/generalSettings', text: 'Settings', icon: <IconSetting /> });
const settingsItems = [
{ itemKey: '/users', text: 'User Management' },
{ itemKey: '/generalSettings', text: 'General Settings' },
];
if (watchlistFeature) {
settingsItems.push({ itemKey: '/watchlistManagement', text: 'Watchlist Management' });
}
items.push({
itemKey: 'settings',
text: 'Settings',
icon: <IconSetting />,
items: settingsItems,
});
}
function parsePathName(name) {
@@ -32,7 +46,7 @@ export default function Navigation({ isAdmin }) {
return (
<Nav
style={{ height: '100%', width: collapsed ? '' : '13rem' }}
style={{ height: '100%', width: collapsed ? '' : '13.2rem' }}
items={items}
isCollapsed={collapsed}
selectedKeys={[parsePathName(location.pathname)]}

View File

@@ -1,7 +1,7 @@
import React from 'react';
import { Button, Empty, Table, Switch, Popover } from '@douyinfe/semi-ui';
import { IconDelete, IconDescend2, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
import { IconAlertTriangle, IconDelete, IconDescend2, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
import './JobTable.less';
@@ -33,12 +33,38 @@ export default function JobTable({
title: '',
dataIndex: '',
render: (job) => {
return <Switch onChange={(checked) => onJobStatusChanged(job.id, checked)} checked={job.enabled} />;
return (
<Switch
onChange={(checked) => onJobStatusChanged(job.id, checked)}
checked={job.enabled}
disabled={job.isOnlyShared}
/>
);
},
},
{
title: 'Name',
dataIndex: 'name',
render: (name, job) => {
if (job.isOnlyShared) {
return (
<Popover
content={getPopoverContent(
'This job has been shared with you by another user, therefor it is read-only.',
)}
>
<div style={{ display: 'flex', gap: '.3rem' }}>
<div style={{ color: 'rgba(var(--semi-yellow-7), 1)' }}>
<IconAlertTriangle />
</div>
{name}
</div>
</Popover>
);
} else {
return name;
}
},
},
{
title: 'Listings',
@@ -48,14 +74,14 @@ export default function JobTable({
},
},
{
title: 'Providers',
title: 'Provider',
dataIndex: 'provider',
render: (value) => {
return value.length || 0;
},
},
{
title: 'Notification adapters',
title: 'Notification Adapter',
dataIndex: 'notificationAdapter',
render: (value) => {
return value.length || 0;
@@ -68,16 +94,36 @@ export default function JobTable({
return (
<div className="interactions">
<Popover content={getPopoverContent('Job Insights')}>
<Button type="primary" icon={<IconHistogram />} onClick={() => onJobInsight(job.id)} />
<Button
type="primary"
icon={<IconHistogram />}
disabled={job.isOnlyShared}
onClick={() => onJobInsight(job.id)}
/>
</Popover>
<Popover content={getPopoverContent('Edit a Job')}>
<Button type="secondary" icon={<IconEdit />} onClick={() => onJobEdit(job.id)} />
<Button
type="secondary"
icon={<IconEdit />}
disabled={job.isOnlyShared}
onClick={() => onJobEdit(job.id)}
/>
</Popover>
<Popover content={getPopoverContent('Delete all found Listings of this Job')}>
<Button type="danger" icon={<IconDescend2 />} onClick={() => onListingRemoval(job.id)} />
<Button
type="danger"
icon={<IconDescend2 />}
disabled={job.isOnlyShared}
onClick={() => onListingRemoval(job.id)}
/>
</Popover>
<Popover content={getPopoverContent('Delete Job')}>
<Button type="danger" icon={<IconDelete />} onClick={() => onJobRemoval(job.id)} />
<Button
type="danger"
icon={<IconDelete />}
disabled={job.isOnlyShared}
onClick={() => onJobRemoval(job.id)}
/>
</Popover>
</div>
);

View File

@@ -1,9 +1,9 @@
import React from 'react';
import { Empty, Table, Button } from '@douyinfe/semi-ui';
import { IconDelete } from '@douyinfe/semi-icons';
import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
export default function ProviderTable({ providerData = [], onRemove } = {}) {
export default function ProviderTable({ providerData = [], onRemove, onEdit } = {}) {
return (
<Table
pagination={false}
@@ -30,6 +30,8 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
render: (_, record) => {
return (
<div style={{ float: 'right' }}>
<Button type="secondary" icon={<IconEdit />} onClick={() => onEdit(record)} />
<div style={{ display: 'inline-block', width: '16px' }} />
<Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
</div>
);

View File

@@ -1,45 +0,0 @@
import { Card, Checkbox, Descriptions, Divider, Select } from '@douyinfe/semi-ui';
import React from 'react';
import { useSelector } from '../../../services/state/store.js';
import { Typography } from '@douyinfe/semi-ui';
import './ListingsFilter.less';
export default function ListingsFilter({ onWatchListFilter, onActivityFilter, onJobNameFilter, onProviderFilter }) {
const jobs = useSelector((state) => state.jobs.jobs);
const provider = useSelector((state) => state.provider);
const { Title } = Typography;
return (
<Card className="listingsFilter">
<Title heading={6}>Filter by:</Title>
<Divider />
<br />
<Descriptions row>
<Descriptions.Item itemKey="Watch List">
<Checkbox onChange={(e) => onWatchListFilter(e.target.checked)}>Only Watch List</Checkbox>
</Descriptions.Item>
<Descriptions.Item itemKey="Activity status">
<Checkbox onChange={(e) => onActivityFilter(e.target.checked)}>Only Active Listings</Checkbox>
</Descriptions.Item>
<Descriptions.Item itemKey="Job Name">
<Select showClear placeholder="Select Job to Filter" onChange={(val) => onJobNameFilter(val)}>
{jobs != null &&
jobs.length > 0 &&
jobs.map((job) => {
return <Select.Option value={job.id}>{job.name}</Select.Option>;
})}
</Select>
</Descriptions.Item>
<Descriptions.Item itemKey="Provider">
<Select showClear placeholder="Select Provider to Filter" onChange={(val) => onProviderFilter(val)}>
{provider != null &&
provider.length > 0 &&
provider.map((prov) => {
return <Select.Option value={prov.id}>{prov.name}</Select.Option>;
})}
</Select>
</Descriptions.Item>
</Descriptions>
</Card>
);
}

View File

@@ -1,4 +0,0 @@
.listingsFilter {
margin-bottom: 1rem;
background: rgb(53, 54, 60);
}

View File

@@ -1,5 +1,18 @@
import React, { useState, useEffect, useMemo } from 'react';
import { Table, Popover, Input, Descriptions, Tag, Image, Empty, Button, Toast, Divider } from '@douyinfe/semi-ui';
import {
Table,
Popover,
Input,
Descriptions,
Tag,
Image,
Empty,
Button,
Toast,
Divider,
Space,
Select,
} from '@douyinfe/semi-ui';
import { useActions, useSelector } from '../../../services/state/store.js';
import { IconClose, IconDelete, IconSearch, IconStar, IconStarStroked, IconTick } from '@douyinfe/semi-icons';
import * as timeService from '../../../services/time/timeService.js';
@@ -10,166 +23,224 @@ import './ListingsTable.less';
import { format } from '../../../services/time/timeService.js';
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
import { xhrDelete, xhrPost } from '../../../services/xhr.js';
import ListingsFilter from './ListingsFilter.jsx';
import { useNavigate } from 'react-router-dom';
import { useFeature } from '../../../hooks/featureHook.js';
const columns = [
{
title: '#',
width: 100,
dataIndex: 'isWatched',
sorter: true,
render: (id, row) => {
return (
<div>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content={row.isWatched === 1 ? 'Unwatch Listing' : 'Watch Listing'}
>
<Button
icon={
row.isWatched === 1 ? (
<IconStar style={{ color: 'rgba(var(--semi-green-5), 1)' }} />
) : (
<IconStarStroked />
)
}
theme="borderless"
size="small"
onClick={async () => {
try {
await xhrPost('/api/listings/watch', { listingId: row.id });
Toast.success(row.isWatched === 1 ? 'Listing removed from Watchlist' : 'Listing added to Watchlist');
row.reloadTable();
} catch (e) {
console.error(e);
Toast.error('Failed to operate Watchlist');
}
const getColumns = (provider, setProviderFilter, jobs, setJobNameFilter) => {
return [
{
title: 'Watchlist',
width: 133,
dataIndex: 'isWatched',
sorter: true,
filters: [
{
text: 'Show only watched listings',
value: 'watchList',
},
],
render: (id, row) => {
return (
<div>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
/>
</Popover>
<Divider layout="vertical" margin="4px" />
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Delete Listing"
>
<Button
icon={<IconDelete />}
theme="borderless"
size="small"
type="danger"
onClick={async () => {
try {
await xhrDelete('/api/listings/', { ids: [row.id] });
Toast.success('Listing(s) successfully removed');
row.reloadTable();
} catch (error) {
Toast.error(error);
content={row.isWatched === 1 ? 'Unwatch Listing' : 'Watch Listing'}
>
<Button
icon={
row.isWatched === 1 ? (
<IconStar style={{ color: 'rgba(var(--semi-green-5), 1)' }} />
) : (
<IconStarStroked />
)
}
theme="borderless"
size="small"
onClick={async () => {
try {
await xhrPost('/api/listings/watch', { listingId: row.id });
Toast.success(
row.isWatched === 1 ? 'Listing removed from Watchlist' : 'Listing added to Watchlist',
);
row.reloadTable();
} catch (e) {
console.error(e);
Toast.error('Failed to operate Watchlist');
}
}}
/>
</Popover>
<Divider layout="vertical" margin="4px" />
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
/>
</Popover>
</div>
);
content="Delete Listing"
>
<Button
icon={<IconDelete />}
theme="borderless"
size="small"
type="danger"
onClick={async () => {
try {
await xhrDelete('/api/listings/', { ids: [row.id] });
Toast.success('Listing(s) successfully removed');
row.reloadTable();
} catch (error) {
Toast.error(error);
}
}}
/>
</Popover>
</div>
);
},
},
},
{
title: 'State',
dataIndex: 'is_active',
width: 84,
sorter: true,
render: (value) => {
return value ? (
<div style={{ color: 'rgba(var(--semi-green-6), 1)' }}>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Listing is still active"
>
<IconTick />
</Popover>
</div>
) : (
<div style={{ color: 'rgba(var(--semi-red-5), 1)' }}>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Listing is inactive"
>
<IconClose />
</Popover>
</div>
);
{
title: 'Active',
dataIndex: 'is_active',
width: 110,
sorter: true,
filters: [
{
text: 'Show only active listings',
value: 'activityStatus',
},
],
render: (value) => {
return value ? (
<div style={{ color: 'rgba(var(--semi-green-6), 1)' }}>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Listing is still active"
>
<IconTick />
</Popover>
</div>
) : (
<div style={{ color: 'rgba(var(--semi-red-5), 1)' }}>
<Popover
style={{
padding: '.4rem',
color: 'var(--semi-color-white)',
}}
content="Listing is inactive"
>
<IconClose />
</Popover>
</div>
);
},
},
},
{
title: 'Job-Name',
sorter: true,
ellipsis: true,
dataIndex: 'job_name',
width: 150,
},
{
title: 'Listing date',
width: 130,
dataIndex: 'created_at',
sorter: true,
render: (text) => timeService.format(text, false),
},
{
title: 'Provider',
width: 130,
dataIndex: 'provider',
sorter: true,
render: (text) => text.charAt(0).toUpperCase() + text.slice(1),
},
{
title: 'Price',
width: 110,
dataIndex: 'price',
sorter: true,
render: (text) => text + ' €',
},
{
title: 'Address',
width: 150,
dataIndex: 'address',
sorter: true,
},
{
title: 'Title',
dataIndex: 'title',
sorter: true,
ellipsis: true,
render: (text, row) => {
return (
<a href={row.url} target="_blank" rel="noopener noreferrer">
{text}
</a>
);
{
title: 'Job-Name',
sorter: true,
ellipsis: true,
dataIndex: 'job_name',
width: 150,
onFilter: () => true,
renderFilterDropdown: () => {
return (
<Space vertical style={{ padding: 8 }}>
<Select showClear placeholder="Select Job to Filter" onChange={(val) => setJobNameFilter(val)}>
{jobs != null &&
jobs.length > 0 &&
jobs.map((job) => {
return (
<Select.Option value={job.id} key={job.id}>
{job.name}
</Select.Option>
);
})}
</Select>
</Space>
);
},
},
},
];
{
title: 'Listing date',
width: 130,
dataIndex: 'created_at',
sorter: true,
render: (text) => timeService.format(text, false),
},
{
title: 'Provider',
width: 130,
dataIndex: 'provider',
sorter: true,
render: (text) => text.charAt(0).toUpperCase() + text.slice(1),
onFilter: () => true,
renderFilterDropdown: () => {
return (
<Space vertical style={{ padding: 8 }}>
<Select showClear placeholder="Select Provider to Filter" onChange={(val) => setProviderFilter(val)}>
{provider != null &&
provider.length > 0 &&
provider.map((prov) => {
return (
<Select.Option value={prov.id} key={prov.id}>
{prov.name}
</Select.Option>
);
})}
</Select>
</Space>
);
},
},
{
title: 'Price',
width: 110,
dataIndex: 'price',
sorter: true,
render: (text) => text + ' €',
},
{
title: 'Address',
width: 150,
dataIndex: 'address',
sorter: true,
},
{
title: 'Title',
dataIndex: 'title',
sorter: true,
ellipsis: true,
render: (text, row) => {
return (
<a href={row.url} target="_blank" rel="noopener noreferrer">
{text}
</a>
);
},
},
];
};
const empty = (
<Empty
image={<IllustrationNoResult />}
darkModeImage={<IllustrationNoResultDark />}
description="No listings available."
description="No listings found."
/>
);
export default function ListingsTable() {
const tableData = useSelector((state) => state.listingsTable);
const provider = useSelector((state) => state.provider);
const jobs = useSelector((state) => state.jobs.jobs);
const navigate = useNavigate();
const watchlistFeature = useFeature('WATCHLIST_MANAGEMENT') || false;
const actions = useActions();
const [page, setPage] = useState(1);
const pageSize = 10;
@@ -179,11 +250,14 @@ export default function ListingsTable() {
const [jobNameFilter, setJobNameFilter] = useState(null);
const [activityFilter, setActivityFilter] = useState(null);
const [providerFilter, setProviderFilter] = useState(null);
const [allFilters, setAllFilters] = useState([]);
const [imageWidth, setImageWidth] = useState('100%');
const handlePageChange = (_page) => {
setPage(_page);
};
const columns = getColumns(provider, setProviderFilter, jobs, setJobNameFilter);
const loadTable = () => {
let sortfield = null;
let sortdir = null;
@@ -208,14 +282,43 @@ export default function ListingsTable() {
const handleFilterChange = useMemo(() => debounce((value) => setFreeTextFilter(value), 500), []);
const diffArrays = (primary, secondary) => {
const result = {};
for (const item of secondary) {
if (!primary.includes(item)) result[item] = true;
}
for (const item of primary) {
if (!secondary.includes(item)) result[item] = false;
}
return [result];
};
useEffect(() => {
return () => {
// cleanup debounced handler to avoid memory leaks
handleFilterChange.cancel && handleFilterChange.cancel();
};
}, [handleFilterChange]);
const expandRowRender = (record) => {
return (
<div className="listingsTable__expanded">
<div>
{record.image_url == null ? (
<Image height={200} src={no_image} />
<Image height={200} width={180} src={no_image} />
) : (
<Image height={200} src={record.image_url} />
<Image
height={200}
width={imageWidth}
src={record.image_url}
onError={() => {
setImageWidth('180px');
}}
fallback={<Image height={200} src={no_image} />}
/>
)}
</div>
<div>
@@ -226,7 +329,7 @@ export default function ListingsTable() {
</Tag>
</Descriptions.Item>
<Descriptions.Item itemKey="Link">
<a href={record.link} target="_blank" rel="noreferrer">
<a href={record.link} target="_blank" rel="noopener noreferrer">
Link to Listing
</a>
</Descriptions.Item>
@@ -242,12 +345,6 @@ export default function ListingsTable() {
return (
<div>
<ListingsFilter
onActivityFilter={setActivityFilter}
onWatchListFilter={setWatchListFilter}
onJobNameFilter={setJobNameFilter}
onProviderFilter={setProviderFilter}
/>
<Input
prefix={<IconSearch />}
showClear
@@ -255,6 +352,16 @@ export default function ListingsTable() {
placeholder="Search"
onChange={handleFilterChange}
/>
{watchlistFeature && (
<Button
className="listingsTable__setupButton"
onClick={() => {
navigate('/watchlistManagement');
}}
>
Setup notifications on watchlist changes
</Button>
)}
<Table
rowKey="id"
empty={empty}
@@ -269,7 +376,23 @@ export default function ListingsTable() {
};
})}
onChange={(changeSet) => {
if (changeSet?.extra?.changeType === 'sorter') {
if (changeSet?.extra?.changeType === 'filter') {
const transformed = changeSet.filters.map((f) => f.dataIndex);
const diff = diffArrays(allFilters, transformed);
setAllFilters(transformed);
diff.forEach((filter) => {
switch (Object.keys(filter)[0]) {
case 'isWatched':
setWatchListFilter(Object.values(filter)[0]);
break;
case 'is_active':
setActivityFilter(Object.values(filter)[0]);
break;
default:
console.error('Unknown filter: ', filter.dataIndex);
}
});
} else if (changeSet?.extra?.changeType === 'sorter') {
setSortData({
field: changeSet.sorter.dataIndex,
direction: changeSet.sorter.sortOrder === 'ascend' ? 'asc' : 'desc',

View File

@@ -11,4 +11,8 @@
&__toolbar {
margin-bottom: 1rem;
}
&__setupButton {
margin-bottom: 1rem;
}
}

View File

@@ -0,0 +1,15 @@
import { useSelector } from '../services/state/store.js';
export function useFeature(name) {
const currentFeatureFlags = useSelector((state) => state.features);
if (Object.keys(currentFeatureFlags || {}).length === 0) {
return null;
}
if (currentFeatureFlags[name] == null) {
console.warn(`Feature flag with name ${name} is unknown.`);
return null;
}
return currentFeatureFlags[name];
}

View File

@@ -48,6 +48,16 @@ export const useFredyState = create(
}
},
},
features: {
async getFeatures() {
try {
const response = await xhrGet('/api/features');
set((state) => ({ ...state.features, ...response.json }));
} catch (Exception) {
console.error('Error while trying to get resource for api/features. Error:', Exception);
}
},
},
provider: {
async getProvider() {
try {
@@ -67,6 +77,14 @@ export const useFredyState = create(
console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
}
},
async getSharableUserList() {
try {
const response = await xhrGet('/api/jobs/shareableUserList');
set((state) => ({ jobs: { ...state.jobs, shareableUserList: Object.freeze(response.json) } }));
} catch (Exception) {
console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
}
},
async getProcessingTimes() {
try {
const response = await xhrGet('/api/jobs/processingTimes');
@@ -168,11 +186,12 @@ export const useFredyState = create(
page: 1,
result: [],
},
features: {},
generalSettings: { settings: {} },
demoMode: { demoMode: false },
versionUpdate: {},
provider: [],
jobs: { jobs: [], insights: {}, processingTimes: {} },
jobs: { jobs: [], insights: {}, processingTimes: {}, shareableUserList: [] },
user: { users: [], currentUser: null },
};
@@ -184,6 +203,7 @@ export const useFredyState = create(
versionUpdate: { ...effects.versionUpdate },
listingsTable: { ...effects.listingsTable },
provider: { ...effects.provider },
features: { ...effects.features },
jobs: { ...effects.jobs },
user: { ...effects.user },
};

View File

@@ -1,16 +1,32 @@
import React from 'react';
import { format } from '../../services/time/timeService';
import { Button, Card, Col, Row, Toast } from '@douyinfe/semi-ui';
import { IconPlayCircle } from '@douyinfe/semi-icons';
import {
IconClock,
IconDoubleChevronLeft,
IconDoubleChevronRight,
IconPlayCircle,
IconSearch,
} from '@douyinfe/semi-icons';
import { xhrPost } from '../../services/xhr.js';
import './ProsessingTimes.less';
import { useScreenWidth } from '../../hooks/screenWidth.js';
function InfoCard({ title, value }) {
function InfoCard({ title, value, icon }) {
const { Meta } = Card;
return (
<Card style={{ maxWidth: '13rem', margin: '1rem', background: 'rgb(53, 54, 60)' }} title={title}>
{value}
</Card>
<div
style={{
margin: '1rem',
background: 'rgb(53, 54, 60)',
borderRadius: '.3rem',
padding: '1rem',
minHeight: '3rem',
}}
>
<Meta title={title} description={value} avatar={icon} />
</div>
);
}
@@ -18,32 +34,57 @@ export default function ProcessingTimes({ processingTimes = {} }) {
if (Object.keys(processingTimes).length === 0) {
return null;
}
const width = useScreenWidth();
const invisible = width <= 1180;
if (invisible) {
return null;
}
return (
<Row>
<Col span={6}>
<InfoCard title="Processing Interval" value={`${processingTimes.interval} min`} />
<InfoCard
title="Search Interval"
value={`${processingTimes.interval} min`}
icon={<IconClock style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
/>
</Col>
{processingTimes.lastRun && (
<>
<Col span={6}>
<InfoCard title="Last run" value={format(processingTimes.lastRun)} />
<InfoCard
title="Last search"
icon={<IconDoubleChevronLeft style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
value={format(processingTimes.lastRun)}
/>
</Col>
<Col span={6}>
<InfoCard title="Next run" value={format(processingTimes.lastRun + processingTimes.interval * 60000)} />
<InfoCard
title="Next search"
icon={<IconDoubleChevronRight style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
value={format(processingTimes.lastRun + processingTimes.interval * 60000)}
/>
</Col>
</>
)}
<Col span={6}>
<InfoCard
title="Find Listings Now"
title="Search Now"
icon={<IconSearch style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
value={
<Button
size="small"
style={{ marginTop: '.2rem' }}
icon={<IconPlayCircle />}
aria-label="Start now"
onClick={async () => {
await xhrPost('/api/jobs/startAll', null);
Toast.success('Successfully triggered Fredy search.');
try {
await xhrPost('/api/jobs/startAll', null);
Toast.success('Successfully triggered Fredy search.');
} catch {
Toast.error('Failed to trigger search');
}
}}
>
Search now

View File

@@ -8,13 +8,22 @@ import Headline from '../../../components/headline/Headline';
import { useActions, useSelector } from '../../../services/state/store';
import { xhrPost } from '../../../services/xhr';
import { useNavigate, useParams } from 'react-router-dom';
import { Divider, Input, Switch, Button, TagInput, Toast } from '@douyinfe/semi-ui';
import { Divider, Input, Switch, Button, TagInput, Toast, Select } from '@douyinfe/semi-ui';
import './JobMutation.less';
import { SegmentPart } from '../../../components/segment/SegmentPart';
import { IconPlusCircle } from '@douyinfe/semi-icons';
import {
IconBell,
IconBriefcase,
IconPaperclip,
IconPlayCircle,
IconPlusCircle,
IconUser,
IconClear,
} from '@douyinfe/semi-icons';
export default function JobMutator() {
const jobs = useSelector((state) => state.jobs.jobs);
const shareableUserList = useSelector((state) => state.jobs.shareableUserList);
const params = useParams();
const jobToBeEdit = params.jobId == null ? null : jobs.find((job) => job.id === params.jobId);
@@ -25,6 +34,7 @@ export default function JobMutator() {
const defaultNotificationAdapter = jobToBeEdit?.notificationAdapter || [];
const defaultEnabled = jobToBeEdit?.enabled ?? true;
const [providerToEdit, setProviderToEdit] = useState(null);
const [providerCreationVisible, setProviderCreationVisibility] = useState(false);
const [notificationCreationVisible, setNotificationCreationVisibility] = useState(false);
const [editNotificationAdapter, setEditNotificationAdapter] = useState(null);
@@ -32,6 +42,7 @@ export default function JobMutator() {
const [name, setName] = useState(defaultName);
const [blacklist, setBlacklist] = useState(defaultBlacklist);
const [notificationAdapterData, setNotificationAdapterData] = useState(defaultNotificationAdapter);
const [shareWithUsers, setShareWithUsers] = useState(jobToBeEdit?.shared_with_user ?? []);
const [enabled, setEnabled] = useState(defaultEnabled);
const navigate = useNavigate();
const actions = useActions();
@@ -40,11 +51,18 @@ export default function JobMutator() {
return Boolean(notificationAdapterData.length && providerData.length && name);
};
const handleProviderEdit = (data) => {
setProviderData(
providerData.map((provider) => (provider.url === data.oldProviderToEdit.url ? data.newData : provider)),
);
};
const mutateJob = async () => {
try {
await xhrPost('/api/jobs', {
provider: providerData,
notificationAdapter: notificationAdapterData,
shareWithUsers,
name,
blacklist,
enabled,
@@ -67,6 +85,8 @@ export default function JobMutator() {
onData={(data) => {
setProviderData([...providerData, data]);
}}
onEditData={handleProviderEdit}
providerToEdit={providerToEdit}
/>
{notificationCreationVisible && (
@@ -91,7 +111,7 @@ export default function JobMutator() {
<Headline text={jobToBeEdit ? 'Edit Job' : 'Create new Job'} />
<form>
<SegmentPart name="Name">
<SegmentPart name="Name" Icon={IconPaperclip}>
<Input
autoFocus
type="text"
@@ -105,7 +125,7 @@ export default function JobMutator() {
<Divider margin="1rem" />
<SegmentPart
name="Providers"
icon="briefcase"
Icon={IconBriefcase}
helpText={`
A provider is essentially the service (e.g. ImmoScout24, Kleinanzeigen) that Fredy searches for new listings.
Fredy will open a new tab pointing to the website of this provider. You have to adjust your search parameter
@@ -116,7 +136,10 @@ export default function JobMutator() {
type="primary"
icon={<IconPlusCircle />}
className="jobMutation__newButton"
onClick={() => setProviderCreationVisibility(true)}
onClick={() => {
setProviderToEdit(null);
setProviderCreationVisibility(true);
}}
>
Add new Provider
</Button>
@@ -126,11 +149,15 @@ export default function JobMutator() {
onRemove={(providerUrl) => {
setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
}}
onEdit={(provider) => {
setProviderCreationVisibility(true);
setProviderToEdit(provider);
}}
/>
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
icon="bell"
Icon={IconBell}
name="Notification Adapters"
helpText="Fredy supports multiple ways to notify you about new findings. These are called notification adapter. You can chose between email, Telegram etc."
>
@@ -157,7 +184,7 @@ export default function JobMutator() {
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
icon="bell"
Icon={IconClear}
name="Blacklist"
helpText="If a listing contains one of these words, it will be filtered out. Type in a word, then hit enter."
>
@@ -169,7 +196,32 @@ export default function JobMutator() {
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
icon="play circle outline"
Icon={IconUser}
name="Sharing with user"
helpText="You can share this job with other users. They will be able to see the listings, but only (as the creator) you can edit the job. Admins are filtered from this list as they have access to everything."
>
{shareableUserList.length === 0 ? (
<div>No users found to share this Job to. Please create additional non-admin user.</div>
) : (
<Select
filter
multiple
placeholder="Search user"
autoClearSearchValue={false}
defaultValue={shareWithUsers}
onChange={(value) => setShareWithUsers(value)}
>
{shareableUserList.map((user) => (
<Select.Option value={user.id} key={user.id}>
{user.name}
</Select.Option>
))}
</Select>
)}
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
Icon={IconPlayCircle}
name="Job activation"
helpText="Whether or not the job is activated. Inactive jobs will be ignored when Fredy checks for new listings."
>

View File

@@ -7,6 +7,7 @@ import { useSelector } from '../../../../../services/state/store';
import { Banner, Button, Form, Modal, Select, Switch } from '@douyinfe/semi-ui';
import './NotificationAdapterMutator.less';
import { useScreenWidth } from '../../../../../hooks/screenWidth.js';
const sortAdapter = (a, b) => {
if (a.name < b.name) {
@@ -21,7 +22,7 @@ const sortAdapter = (a, b) => {
const validate = (selectedAdapter) => {
const results = [];
for (let uiElement of Object.values(selectedAdapter.fields || [])) {
if (uiElement.value == null) {
if (uiElement.value == null && !uiElement.optional) {
results.push('All fields are mandatory and must be set.');
continue;
}
@@ -36,7 +37,7 @@ const validate = (selectedAdapter) => {
results.push('A boolean field cannot be of a different type.');
continue;
}
if (typeof uiElement.value === 'string' && uiElement.value.length === 0) {
if (typeof uiElement.value === 'string' && uiElement.value.length === 0 && !uiElement.optional) {
results.push('All fields are mandatory and must be set.');
}
}
@@ -53,6 +54,8 @@ function spreadPrefilledAdapterWithValues(prefilled, fields) {
}
export default function NotificationAdapterMutator({
title,
description,
onVisibilityChanged,
visible = false,
selected = [],
@@ -70,6 +73,9 @@ export default function NotificationAdapterMutator({
const [validationMessage, setValidationMessage] = useState(null);
const [successMessage, setSuccessMessage] = useState(null);
const width = useScreenWidth();
const isMobile = width <= 850;
const onSubmit = (doStore) => {
if (doStore) {
const validationResults = validate(selectedAdapter);
@@ -168,20 +174,21 @@ export default function NotificationAdapterMutator({
return (
<Modal
title="Adding a new Notification Adapter"
title={title != null ? title : 'Adding a new Notification Adapter'}
visible={visible}
style={{ width: '95%' }}
style={{ width: isMobile ? '95%' : '50rem' }}
onCancel={() => onSubmit(false)}
footer={
<div>
<Button type="secondary" disabled={selectedAdapter == null} style={{ float: 'left' }} onClick={() => onTry()}>
<Button type="secondary" disabled={selectedAdapter == null} style={{ float: 'left' }} onClick={onTry}>
Try
</Button>
<Button type="danger" onClick={() => onSubmit(true)}>
Save
</Button>
<Button type="primary" onClick={() => onSubmit(false)}>
<Button theme="light" type="tertiary" onClick={() => onSubmit(false)}>
Cancel
</Button>
<Button theme="solid" type="primary" onClick={() => onSubmit(true)}>
Save
</Button>
</div>
}
>
@@ -206,11 +213,15 @@ export default function NotificationAdapterMutator({
/>
)}
<p>
When Fredy found new listings, we like to report them to you. To do so, notification adapter can be configured.{' '}
<br />
There are multiple ways how Fredy can send new listings to you. Chose your weapon...
</p>
{description != null ? (
<p>{description}</p>
) : (
<p>
When Fredy finds new listings, we like to report them to you. To do so, notification adapter can be
configured. <br />
There are multiple ways how Fredy can send new listings to you. Chose your weapon...
</p>
)}
<Select
filter

View File

@@ -1,5 +1,5 @@
import React from 'react';
import { Banner } from '@douyinfe/semi-ui';
import { Banner, MarkdownRender } from '@douyinfe/semi-ui';
export default function Help({ readme }) {
return (
@@ -8,7 +8,7 @@ export default function Help({ readme }) {
type="info"
closeIcon={null}
title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Information</div>}
description={<p dangerouslySetInnerHTML={{ __html: readme }} />}
description={<MarkdownRender raw={readme} />}
/>
);
}

View File

@@ -1,10 +1,11 @@
import React, { useState } from 'react';
import React, { useState, useEffect } from 'react';
import { Banner, Modal, Select, Input } from '@douyinfe/semi-ui';
import { transform } from '../../../../../services/transformer/providerTransformer';
import { useSelector } from '../../../../../services/state/store';
import { IconLikeHeart } from '@douyinfe/semi-icons';
import './ProviderMutator.less';
import { useScreenWidth } from '../../../../../hooks/screenWidth.js';
const sortProvider = (a, b) => {
if (a.key < b.key) {
@@ -16,11 +17,35 @@ const sortProvider = (a, b) => {
return 0;
};
export default function ProviderMutator({ onVisibilityChanged, visible = false, onData } = {}) {
const returnOriginalSelectedProvider = (providerToEdit, provider) => {
return provider.find((pro) => pro.id === providerToEdit.id);
};
export default function ProviderMutator({
onVisibilityChanged,
visible = false,
onData,
onEditData,
providerToEdit,
} = {}) {
const provider = useSelector((state) => state.provider);
const [selectedProvider, setSelectedProvider] = useState(null);
const [providerUrl, setProviderUrl] = useState(null);
const [validationMessage, setValidationMessage] = useState(null);
useEffect(() => {
if (providerToEdit) {
setSelectedProvider(returnOriginalSelectedProvider(providerToEdit, provider));
setProviderUrl(providerToEdit.url);
} else {
setSelectedProvider(null);
setProviderUrl(null);
}
}, [providerToEdit, visible]);
const width = useScreenWidth();
const isMobile = width <= 850;
const validate = () => {
if (selectedProvider == null || selectedProvider.length === 0 || providerUrl == null || providerUrl.length === 0) {
return 'Please select a provider and copy the browser url into the textfield after configuring your search parameter.';
@@ -41,13 +66,24 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
if (doStore) {
const validationResult = validate();
if (validationResult == null) {
onData(
transform({
url: providerUrl,
id: selectedProvider.id,
name: selectedProvider.name,
}),
);
if (providerToEdit != null) {
onEditData({
newData: transform({
url: providerUrl,
id: selectedProvider.id,
name: selectedProvider.name,
}),
oldProviderToEdit: providerToEdit,
});
} else {
onData(
transform({
url: providerUrl,
id: selectedProvider.id,
name: selectedProvider.name,
}),
);
}
setProviderUrl(null);
setSelectedProvider(null);
onVisibilityChanged(false);
@@ -63,11 +99,11 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
return (
<Modal
title="Adding a new Provider"
title={providerToEdit ? 'Editing an existing Provider' : 'Adding a new Provider'}
visible={visible}
onOk={() => onSubmit(true)}
onCancel={() => onSubmit(false)}
style={{ width: '50rem' }}
style={{ width: isMobile ? '95%' : '50rem' }}
okText="Save"
>
{validationMessage != null && (
@@ -80,19 +116,26 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
description={validationMessage}
/>
)}
<p>
Provider are the <IconLikeHeart style={{ color: '#ff0000' }} /> of Fredy. We're supporting multiple Provider
such as Immowelt, Kalaydo etc. Select a provider from the list below.
<br />
Fredy will then open the provider's url in a new tab.
</p>
<p>
You will need to configure your search parameter like you would do when you do a regular search on the
provider's website.
<br />
When the search results are shown on the website, copy the url and paste it into the textfield below.
</p>
{providerToEdit != null ? (
<p>
You can now edit the <strong>{providerToEdit.name}</strong> provider's URL in the input field below.
</p>
) : (
<>
<p>
Provider are the <IconLikeHeart style={{ color: '#ff0000' }} /> of Fredy. We're supporting multiple Provider
such as Immowelt, Kalaydo etc. Select a provider from the list below.
<br />
Fredy will then open the provider's url in a new tab.
</p>
<p>
You will need to configure your search parameter like you would do when you do a regular search on the
provider's website.
<br />
When the search results are shown on the website, copy the url and paste it into the textfield below.
</p>
</>
)}
<Banner
fullMode={false}
type="warning"
@@ -112,6 +155,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
filter
placeholder="Select a provider"
className="providerMutator__fields"
disabled={providerToEdit != null}
optionList={provider
.map((pro) => {
return {
@@ -126,7 +170,6 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
onChange={(value) => {
const selectedProvider = provider.find((pro) => pro.id === value);
setSelectedProvider(selectedProvider);
window.open(selectedProvider.baseUrl);
}}
/>
@@ -137,7 +180,8 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
placeholder="Provider Url"
width={10}
className="providerMutator__fields"
onBlur={(e) => {
value={providerUrl}
onInput={(e) => {
setProviderUrl(e.target.value);
}}
/>

View File

@@ -3,9 +3,5 @@ import React from 'react';
import ListingsTable from '../../components/table/listings/ListingsTable.jsx';
export default function Listings() {
return (
<div>
<ListingsTable />
</div>
);
return <ListingsTable />;
}

View File

@@ -0,0 +1,59 @@
import React, { useState } from 'react';
import { IconHorn } from '@douyinfe/semi-icons';
import { SegmentPart } from '../../../components/segment/SegmentPart.jsx';
import { Banner, Button, Checkbox, Space } from '@douyinfe/semi-ui';
import NotificationAdapterMutator from '../../jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx';
import Headline from '../../../components/headline/Headline.jsx';
export default function WatchlistManagement() {
const [notificationChooserVisible, setNotificationChooserVisible] = useState(false);
const [notificationAdapterData, setNotificationAdapterData] = useState([]);
//TODO: Set default
const [activityChanges, setActivityChanges] = useState(false);
const [priceChanges, setPriceChanges] = useState(false);
return (
<div>
<SegmentPart
name="Notification for Watch List"
helpText="You can get notified for changes on listings from your watch list."
Icon={IconHorn}
>
<Banner
fullMode={false}
type="info"
closeIcon={null}
title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Note</div>}
description="Youll receive notifications only for listings that are on your watch list. To add listings to it, open the 'Listings' section and tag the ones you want to follow."
/>
<Space />
<Headline size={5} text="Notify me when:" style={{ marginTop: '1rem' }} />
<Checkbox checked={activityChanges} onChange={(e) => setActivityChanges(e.target.checked)}>
Listing state changes (e.g. listing becomes inactive)
</Checkbox>
<Checkbox checked={priceChanges} onChange={(e) => setPriceChanges(e.target.checked)}>
Listing price changes
</Checkbox>
<Space />
<Headline size={5} text="Notify me with:" style={{ marginTop: '1rem' }} />
<Button onClick={() => setNotificationChooserVisible(true)}>Select notification method</Button>
<NotificationAdapterMutator
title="Add notification method"
description="When something has changed, Fredy will notify you using the selected notification adapter. Note, some adapter like SqLite are not available here."
visible={notificationChooserVisible}
onVisibilityChanged={(visible) => {
setNotificationChooserVisible(visible);
}}
selected={notificationAdapterData}
editNotificationAdapter={null}
onData={(data) => {
const oldData = [...notificationAdapterData].filter((o) => o.id !== data.id);
setNotificationAdapterData([...oldData, data]);
}}
/>
</SegmentPart>
</div>
);
}

1283
yarn.lock

File diff suppressed because it is too large Load Diff