mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Feature/spec filter (#276)
* feat(): create map component, add area filtering to the job config * feat(): filter listings by area filter * chore(): cleanup * feat(): solve feedback * feat(): solve most providers * feat(): solve maybe other providers * feat(): add specFilter config, also add rooms to listing * feat(): change tests * feat(): fix kleinanzeigen parser * feat(): add spec filter switch for listing overviiews * feat(): add rooms and size to the overview and detail of a listing * feat(): rem label * feat(): add types, update providers, they now return specs as numbers * feat(): add jsonconfig to enable type checks * feat: add type for prividerConfig, add fieldNames per provider * feat: fix tests, provider, add formatListing * chore: remov duplicates * feat(): fix tests * feat: fix immoscout * chore: geojson typing * feat: solve requested changes
This commit is contained in:
@@ -178,15 +178,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
||||
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
|
||||
}
|
||||
|
||||
await new FredyPipelineExecutioner(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
job.spatialFilter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
browser,
|
||||
).execute();
|
||||
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ export const upsertJob = ({
|
||||
userId,
|
||||
shareWithUsers = [],
|
||||
spatialFilter = null,
|
||||
specFilter = null,
|
||||
}) => {
|
||||
const id = jobId || nanoid();
|
||||
const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0];
|
||||
@@ -44,7 +45,8 @@ export const upsertJob = ({
|
||||
provider = @provider,
|
||||
notification_adapter = @notification_adapter,
|
||||
shared_with_user = @shareWithUsers,
|
||||
spatial_filter = @spatialFilter
|
||||
spatial_filter = @spatialFilter,
|
||||
spec_filter = @specFilter
|
||||
WHERE id = @id`,
|
||||
{
|
||||
id,
|
||||
@@ -55,12 +57,13 @@ export const upsertJob = ({
|
||||
provider: toJson(provider ?? []),
|
||||
notification_adapter: toJson(notificationAdapter ?? []),
|
||||
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
|
||||
specFilter: specFilter ? toJson(specFilter) : null,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
SqliteConnection.execute(
|
||||
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter)
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`,
|
||||
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter, spec_filter)
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter, @specFilter)`,
|
||||
{
|
||||
id,
|
||||
user_id: ownerId,
|
||||
@@ -71,6 +74,7 @@ export const upsertJob = ({
|
||||
shareWithUsers: toJson(shareWithUsers ?? []),
|
||||
notification_adapter: toJson(notificationAdapter ?? []),
|
||||
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
|
||||
specFilter: specFilter ? toJson(specFilter) : null,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -92,6 +96,7 @@ export const getJob = (jobId) => {
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
j.spatial_filter AS spatialFilter,
|
||||
j.spec_filter AS specFilter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
WHERE j.id = @id
|
||||
@@ -107,6 +112,7 @@ export const getJob = (jobId) => {
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
spatialFilter: fromJson(row.spatialFilter, null),
|
||||
specFilter: fromJson(row.specFilter, null),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -157,6 +163,7 @@ export const getJobs = () => {
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
j.spatial_filter AS spatialFilter,
|
||||
j.spec_filter AS specFilter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
WHERE j.enabled = 1
|
||||
@@ -170,6 +177,7 @@ export const getJobs = () => {
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
spatialFilter: fromJson(row.spatialFilter, null),
|
||||
specFilter: fromJson(row.specFilter, null),
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -260,6 +268,7 @@ export const queryJobs = ({
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
j.spatial_filter AS spatialFilter,
|
||||
j.spec_filter AS specFilter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
${whereSql}
|
||||
@@ -276,6 +285,7 @@ export const queryJobs = ({
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
spatialFilter: fromJson(row.spatialFilter, null),
|
||||
specFilter: fromJson(row.specFilter, null),
|
||||
}));
|
||||
|
||||
return { totalNumber, page: safePage, result };
|
||||
|
||||
@@ -174,9 +174,9 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, rooms, title, image_url, description, address,
|
||||
link, created_at, is_active, latitude, longitude)
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @rooms, @title, @image_url, @description, @address, @link,
|
||||
@created_at, 1, @latitude, @longitude)
|
||||
ON CONFLICT(job_id, hash) DO NOTHING`,
|
||||
);
|
||||
@@ -187,8 +187,9 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
hash: item.id,
|
||||
provider: providerId,
|
||||
job_id: jobId,
|
||||
price: extractNumber(item.price),
|
||||
size: extractNumber(item.size),
|
||||
price: item.price,
|
||||
size: item.size,
|
||||
rooms: item.rooms,
|
||||
title: item.title,
|
||||
image_url: item.image,
|
||||
description: item.description,
|
||||
@@ -202,19 +203,6 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Extract the first number from a string like "1.234 €" or "70 m²".
|
||||
* Removes dots/commas before parsing. Returns null on invalid input.
|
||||
* @param {string|undefined|null} str
|
||||
* @returns {number|null}
|
||||
*/
|
||||
function extractNumber(str) {
|
||||
if (!str) return null;
|
||||
const cleaned = str.replace(/\./g, '').replace(',', '.');
|
||||
const num = parseFloat(cleaned);
|
||||
return isNaN(num) ? null : num;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove any parentheses segments (including surrounding whitespace) from a string.
|
||||
* Returns null for empty input.
|
||||
|
||||
10
lib/services/storage/migrations/sql/15.add-listing-specs.js
Normal file
10
lib/services/storage/migrations/sql/15.add-listing-specs.js
Normal file
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE jobs ADD COLUMN spec_filter JSONB DEFAULT NULL;
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN rooms INTEGER;
|
||||
`);
|
||||
}
|
||||
Reference in New Issue
Block a user