mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b56e13aa16 | ||
|
|
a834abc31c | ||
|
|
573868eccb |
@@ -5,9 +5,10 @@
|
|||||||
|
|
||||||
import { NoNewListingsWarning } from './errors.js';
|
import { NoNewListingsWarning } from './errors.js';
|
||||||
import {
|
import {
|
||||||
storeListings,
|
|
||||||
getKnownListingHashesForJobAndProvider,
|
|
||||||
deleteListingsById,
|
deleteListingsById,
|
||||||
|
getKnownListingHashesForJobAndProvider,
|
||||||
|
storeListings,
|
||||||
|
updateListingDistance,
|
||||||
} from './services/storage/listingsStorage.js';
|
} from './services/storage/listingsStorage.js';
|
||||||
import { getJob } from './services/storage/jobStorage.js';
|
import { getJob } from './services/storage/jobStorage.js';
|
||||||
import * as notify from './notification/notify.js';
|
import * as notify from './notification/notify.js';
|
||||||
@@ -16,8 +17,7 @@ import urlModifier from './services/queryStringMutator.js';
|
|||||||
import logger from './services/logger.js';
|
import logger from './services/logger.js';
|
||||||
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
||||||
import { distanceMeters } from './services/listings/distanceCalculator.js';
|
import { distanceMeters } from './services/listings/distanceCalculator.js';
|
||||||
import { getUserSettings, getSettings } from './services/storage/settingsStorage.js';
|
import { getSettings, getUserSettings } from './services/storage/settingsStorage.js';
|
||||||
import { updateListingDistance } from './services/storage/listingsStorage.js';
|
|
||||||
import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
|
import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
|
||||||
import { formatListing } from './utils/formatListing.js';
|
import { formatListing } from './utils/formatListing.js';
|
||||||
|
|
||||||
@@ -97,9 +97,9 @@ class FredyPipelineExecutioner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Optionally enrich new listings with data from their detail pages.
|
* Optionally, enrich new listings with data from their detail pages.
|
||||||
* Only called when the provider config defines a `fetchDetails` function.
|
* Only called when the provider config defines a `fetchDetails` function.
|
||||||
* Runs all fetches in parallel. Each individual fetch must handle its own errors
|
* Runs all fetches in parallel. Each fetch must handle its own errors
|
||||||
* and always resolve (never reject) to avoid aborting other listings.
|
* and always resolve (never reject) to avoid aborting other listings.
|
||||||
*
|
*
|
||||||
* @param {Listing[]} newListings New listings to enrich.
|
* @param {Listing[]} newListings New listings to enrich.
|
||||||
@@ -132,7 +132,7 @@ class FredyPipelineExecutioner {
|
|||||||
for (const listing of newListings) {
|
for (const listing of newListings) {
|
||||||
if (listing.address) {
|
if (listing.address) {
|
||||||
const coords = await geocodeAddress(listing.address);
|
const coords = await geocodeAddress(listing.address);
|
||||||
if (coords) {
|
if (coords && coords.lat !== -1 && coords.lng !== -1) {
|
||||||
listing.latitude = coords.lat;
|
listing.latitude = coords.lat;
|
||||||
listing.longitude = coords.lng;
|
listing.longitude = coords.lng;
|
||||||
}
|
}
|
||||||
@@ -264,15 +264,15 @@ class FredyPipelineExecutioner {
|
|||||||
const requiredKeys = this._providerConfig.requiredFieldNames;
|
const requiredKeys = this._providerConfig.requiredFieldNames;
|
||||||
const requireValues = ['id', 'link', 'title'];
|
const requireValues = ['id', 'link', 'title'];
|
||||||
|
|
||||||
const filteredListings = listings
|
return (
|
||||||
// this should never filter some listings out, because the normalize function should always extract all fields.
|
listings
|
||||||
.filter((item) => requiredKeys.every((key) => key in item))
|
// this should never filter some listings out, because the normalize function should always extract all fields.
|
||||||
// TODO: move blacklist filter to this file, so it will handle for all providers in same way.
|
.filter((item) => requiredKeys.every((key) => key in item))
|
||||||
.filter(this._providerConfig.filter)
|
// TODO: move blacklist filter to this file, so it will handle for all providers in same way.
|
||||||
// filter out listings that are missing required fields
|
.filter(this._providerConfig.filter)
|
||||||
.filter((item) => requireValues.every((key) => item[key] != null));
|
// filter out listings that are missing required fields
|
||||||
|
.filter((item) => requireValues.every((key) => item[key] != null))
|
||||||
return filteredListings;
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -214,6 +214,8 @@ export const storeListings = (jobId, providerId, listings) => {
|
|||||||
longitude: item.longitude || null,
|
longitude: item.longitude || null,
|
||||||
};
|
};
|
||||||
stmt.run(params);
|
stmt.run(params);
|
||||||
|
// Propagate the DB primary key back so downstream pipeline steps use the correct id
|
||||||
|
item.id = params.id;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -417,9 +419,10 @@ export const deleteListingsByJobId = (jobId, hardDelete = false) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete listings by a list of listing IDs.
|
* Delete listings by a list of listing IDs (the nanoid primary key stored in the `id` column).
|
||||||
|
* Used by API routes that receive row IDs from the client.
|
||||||
*
|
*
|
||||||
* @param {string[]} ids - Array of listing IDs to delete.
|
* @param {string[]} ids - Array of DB row IDs to delete.
|
||||||
* @param {boolean} [hardDelete=false] - Whether to hard delete from DB or just mark as deleted.
|
* @param {boolean} [hardDelete=false] - Whether to hard delete from DB or just mark as deleted.
|
||||||
* @returns {any} The result from SqliteConnection.execute.
|
* @returns {any} The result from SqliteConnection.execute.
|
||||||
*/
|
*/
|
||||||
|
|||||||
44
package.json
44
package.json
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "fredy",
|
"name": "fredy",
|
||||||
"version": "22.1.1",
|
"version": "22.2.0",
|
||||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"prepare": "husky",
|
"prepare": "husky",
|
||||||
@@ -62,9 +62,9 @@
|
|||||||
"Firefox ESR"
|
"Firefox ESR"
|
||||||
],
|
],
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@douyinfe/semi-icons": "^2.99.2",
|
"@douyinfe/semi-icons": "^2.99.3",
|
||||||
"@douyinfe/semi-ui": "2.99.2",
|
"@douyinfe/semi-ui": "2.99.3",
|
||||||
"@douyinfe/semi-ui-19": "^2.99.2",
|
"@douyinfe/semi-ui-19": "^2.99.3",
|
||||||
"@fastify/cookie": "^11.0.2",
|
"@fastify/cookie": "^11.0.2",
|
||||||
"@fastify/helmet": "^13.0.2",
|
"@fastify/helmet": "^13.0.2",
|
||||||
"@fastify/session": "^11.1.1",
|
"@fastify/session": "^11.1.1",
|
||||||
@@ -78,7 +78,7 @@
|
|||||||
"better-sqlite3": "^12.10.0",
|
"better-sqlite3": "^12.10.0",
|
||||||
"chart.js": "^4.5.1",
|
"chart.js": "^4.5.1",
|
||||||
"cheerio": "^1.2.0",
|
"cheerio": "^1.2.0",
|
||||||
"cloakbrowser": "^0.3.30",
|
"cloakbrowser": "^0.3.31",
|
||||||
"fastify": "^5.8.5",
|
"fastify": "^5.8.5",
|
||||||
"handlebars": "4.7.9",
|
"handlebars": "4.7.9",
|
||||||
"maplibre-gl": "^5.24.0",
|
"maplibre-gl": "^5.24.0",
|
||||||
@@ -86,41 +86,41 @@
|
|||||||
"node-cron": "^4.2.1",
|
"node-cron": "^4.2.1",
|
||||||
"node-fetch": "3.3.2",
|
"node-fetch": "3.3.2",
|
||||||
"node-mailjet": "6.0.11",
|
"node-mailjet": "6.0.11",
|
||||||
"nodemailer": "^8.0.7",
|
"nodemailer": "^8.0.10",
|
||||||
"p-throttle": "^8.1.0",
|
"p-throttle": "^8.1.0",
|
||||||
"package-up": "^5.0.0",
|
"package-up": "^5.0.0",
|
||||||
"puppeteer-core": "^25.0.4",
|
"puppeteer-core": "^25.1.0",
|
||||||
"query-string": "9.3.1",
|
"query-string": "9.4.0",
|
||||||
"react": "19.2.6",
|
"react": "19.2.7",
|
||||||
"react-chartjs-2": "^5.3.1",
|
"react-chartjs-2": "^5.3.1",
|
||||||
"react-dom": "19.2.6",
|
"react-dom": "19.2.7",
|
||||||
"react-range-slider-input": "^3.3.5",
|
"react-range-slider-input": "^3.3.5",
|
||||||
"react-router": "7.15.1",
|
"react-router": "7.16.0",
|
||||||
"react-router-dom": "7.15.1",
|
"react-router-dom": "7.16.0",
|
||||||
"resend": "^6.12.3",
|
"resend": "^6.12.4",
|
||||||
"semver": "^7.8.1",
|
"semver": "^7.8.1",
|
||||||
"slack": "11.0.2",
|
"slack": "11.0.2",
|
||||||
"vite": "8.0.14",
|
"vite": "8.0.16",
|
||||||
"x-var": "^3.0.1",
|
"x-var": "^3.0.1",
|
||||||
"zustand": "^5.0.13"
|
"zustand": "^5.0.14"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/core": "7.29.0",
|
"@babel/core": "7.29.7",
|
||||||
"@babel/eslint-parser": "7.28.6",
|
"@babel/eslint-parser": "7.29.7",
|
||||||
"@babel/preset-env": "7.29.5",
|
"@babel/preset-env": "7.29.7",
|
||||||
"@babel/preset-react": "7.28.5",
|
"@babel/preset-react": "7.29.7",
|
||||||
"@eslint/js": "^10.0.1",
|
"@eslint/js": "^10.0.1",
|
||||||
"chalk": "^5.6.2",
|
"chalk": "^5.6.2",
|
||||||
"eslint": "10.4.0",
|
"eslint": "10.4.1",
|
||||||
"eslint-config-prettier": "10.1.8",
|
"eslint-config-prettier": "10.1.8",
|
||||||
"eslint-plugin-react": "7.37.5",
|
"eslint-plugin-react": "7.37.5",
|
||||||
"globals": "^17.6.0",
|
"globals": "^17.6.0",
|
||||||
"history": "5.3.0",
|
"history": "5.3.0",
|
||||||
"husky": "9.1.7",
|
"husky": "9.1.7",
|
||||||
"less": "4.6.4",
|
"less": "4.6.4",
|
||||||
"lint-staged": "17.0.5",
|
"lint-staged": "17.0.7",
|
||||||
"nodemon": "^3.1.14",
|
"nodemon": "^3.1.14",
|
||||||
"prettier": "3.8.3",
|
"prettier": "3.8.3",
|
||||||
"vitest": "^4.1.7"
|
"vitest": "^4.1.8"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,4 +32,7 @@ export const deletedIds = [];
|
|||||||
export const deleteListingsById = (ids) => {
|
export const deleteListingsById = (ids) => {
|
||||||
deletedIds.push(...ids);
|
deletedIds.push(...ids);
|
||||||
};
|
};
|
||||||
|
export const deleteListingsByHash = (hashes) => {
|
||||||
|
deletedIds.push(...hashes);
|
||||||
|
};
|
||||||
/* eslint-enable no-unused-vars */
|
/* eslint-enable no-unused-vars */
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import maplibregl from 'maplibre-gl';
|
|||||||
import 'maplibre-gl/dist/maplibre-gl.css';
|
import 'maplibre-gl/dist/maplibre-gl.css';
|
||||||
import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css';
|
import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css';
|
||||||
import { fixMapboxDrawCompatibility, addDrawingControl, setupAreaFilterEventListeners } from './MapDrawingExtension.js';
|
import { fixMapboxDrawCompatibility, addDrawingControl, setupAreaFilterEventListeners } from './MapDrawingExtension.js';
|
||||||
|
import { getBoundsFromCoords } from '../../views/listings/mapUtils.js';
|
||||||
import './Map.less';
|
import './Map.less';
|
||||||
|
|
||||||
export const GERMANY_BOUNDS = [
|
export const GERMANY_BOUNDS = [
|
||||||
@@ -66,6 +67,7 @@ export default function Map({
|
|||||||
const mapContainerRef = useRef(null);
|
const mapContainerRef = useRef(null);
|
||||||
const mapRef = useRef(null);
|
const mapRef = useRef(null);
|
||||||
const drawRef = useRef(null);
|
const drawRef = useRef(null);
|
||||||
|
const hasFittedToInitialAreaRef = useRef(false);
|
||||||
|
|
||||||
// Initialize map - ONLY when container changes, never reinitialize
|
// Initialize map - ONLY when container changes, never reinitialize
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -128,6 +130,17 @@ export default function Map({
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error loading spatial filter:', error);
|
console.error('Error loading spatial filter:', error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!hasFittedToInitialAreaRef.current) {
|
||||||
|
const coords = initialSpatialFilter.features.flatMap((feature) =>
|
||||||
|
feature.geometry?.type === 'Polygon' ? feature.geometry.coordinates.flat() : [],
|
||||||
|
);
|
||||||
|
const bounds = getBoundsFromCoords(coords);
|
||||||
|
if (bounds) {
|
||||||
|
mapRef.current.fitBounds(bounds, { padding: 50, maxZoom: 15, duration: 0 });
|
||||||
|
hasFittedToInitialAreaRef.current = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Setup drawing event listeners
|
// Setup drawing event listeners
|
||||||
|
|||||||
Reference in New Issue
Block a user