Feature/spec filter (#276)

* feat(): create map component, add area filtering to the job config

* feat(): filter listings by area filter

* chore(): cleanup

* feat(): solve feedback

* feat(): solve most providers

* feat(): solve maybe other providers

* feat(): add specFilter config, also add rooms to listing

* feat(): change tests

* feat(): fix kleinanzeigen parser

* feat(): add spec filter switch for listing overviiews

* feat(): add rooms and size to the overview and detail of a listing

* feat(): rem label

* feat(): add types, update providers, they now return specs as numbers

* feat(): add jsonconfig to enable type checks

* feat: add type for prividerConfig, add fieldNames per provider

* feat: fix tests, provider, add formatListing

* chore: remov duplicates

* feat(): fix tests

* feat: fix immoscout

* chore: geojson typing

* feat: solve requested changes
This commit is contained in:
Stephan
2026-04-12 09:17:23 +02:00
committed by GitHub
parent 05f74f99ef
commit 10c94eea0a
49 changed files with 1004 additions and 250 deletions

View File

@@ -178,15 +178,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
}
await new FredyPipelineExecutioner(
matchedProvider.config,
job.notificationAdapter,
job.spatialFilter,
prov.id,
job.id,
similarityCache,
browser,
).execute();
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();
} catch (err) {
logger.error(err);
}

View File

@@ -31,6 +31,7 @@ export const upsertJob = ({
userId,
shareWithUsers = [],
spatialFilter = null,
specFilter = null,
}) => {
const id = jobId || nanoid();
const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0];
@@ -44,7 +45,8 @@ export const upsertJob = ({
provider = @provider,
notification_adapter = @notification_adapter,
shared_with_user = @shareWithUsers,
spatial_filter = @spatialFilter
spatial_filter = @spatialFilter,
spec_filter = @specFilter
WHERE id = @id`,
{
id,
@@ -55,12 +57,13 @@ export const upsertJob = ({
provider: toJson(provider ?? []),
notification_adapter: toJson(notificationAdapter ?? []),
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
specFilter: specFilter ? toJson(specFilter) : null,
},
);
} else {
SqliteConnection.execute(
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter)
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`,
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter, spec_filter)
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter, @specFilter)`,
{
id,
user_id: ownerId,
@@ -71,6 +74,7 @@ export const upsertJob = ({
shareWithUsers: toJson(shareWithUsers ?? []),
notification_adapter: toJson(notificationAdapter ?? []),
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
specFilter: specFilter ? toJson(specFilter) : null,
},
);
}
@@ -92,6 +96,7 @@ export const getJob = (jobId) => {
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
j.spatial_filter AS spatialFilter,
j.spec_filter AS specFilter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
FROM jobs j
WHERE j.id = @id
@@ -107,6 +112,7 @@ export const getJob = (jobId) => {
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
spatialFilter: fromJson(row.spatialFilter, null),
specFilter: fromJson(row.specFilter, null),
};
};
@@ -157,6 +163,7 @@ export const getJobs = () => {
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
j.spatial_filter AS spatialFilter,
j.spec_filter AS specFilter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
FROM jobs j
WHERE j.enabled = 1
@@ -170,6 +177,7 @@ export const getJobs = () => {
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
spatialFilter: fromJson(row.spatialFilter, null),
specFilter: fromJson(row.specFilter, null),
}));
};
@@ -260,6 +268,7 @@ export const queryJobs = ({
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
j.spatial_filter AS spatialFilter,
j.spec_filter AS specFilter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
FROM jobs j
${whereSql}
@@ -276,6 +285,7 @@ export const queryJobs = ({
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
spatialFilter: fromJson(row.spatialFilter, null),
specFilter: fromJson(row.specFilter, null),
}));
return { totalNumber, page: safePage, result };

View File

@@ -174,9 +174,9 @@ export const storeListings = (jobId, providerId, listings) => {
SqliteConnection.withTransaction((db) => {
const stmt = db.prepare(
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
`INSERT INTO listings (id, hash, provider, job_id, price, size, rooms, title, image_url, description, address,
link, created_at, is_active, latitude, longitude)
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
VALUES (@id, @hash, @provider, @job_id, @price, @size, @rooms, @title, @image_url, @description, @address, @link,
@created_at, 1, @latitude, @longitude)
ON CONFLICT(job_id, hash) DO NOTHING`,
);
@@ -187,8 +187,9 @@ export const storeListings = (jobId, providerId, listings) => {
hash: item.id,
provider: providerId,
job_id: jobId,
price: extractNumber(item.price),
size: extractNumber(item.size),
price: item.price,
size: item.size,
rooms: item.rooms,
title: item.title,
image_url: item.image,
description: item.description,
@@ -202,19 +203,6 @@ export const storeListings = (jobId, providerId, listings) => {
}
});
/**
* Extract the first number from a string like "1.234 €" or "70 m²".
* Removes dots/commas before parsing. Returns null on invalid input.
* @param {string|undefined|null} str
* @returns {number|null}
*/
function extractNumber(str) {
if (!str) return null;
const cleaned = str.replace(/\./g, '').replace(',', '.');
const num = parseFloat(cleaned);
return isNaN(num) ? null : num;
}
/**
* Remove any parentheses segments (including surrounding whitespace) from a string.
* Returns null for empty input.

View File

@@ -0,0 +1,10 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export function up(db) {
db.exec(`
ALTER TABLE jobs ADD COLUMN spec_filter JSONB DEFAULT NULL;
`);
}

View File

@@ -0,0 +1,10 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export function up(db) {
db.exec(`
ALTER TABLE listings ADD COLUMN rooms INTEGER;
`);
}