From c839f3abc9b9a7ff17308cb47222233c45b980c5 Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Mon, 22 Sep 2025 09:57:50 +0200 Subject: [PATCH] Check if a listing is still active (#184) * check if a listing is still active * upgrade dependencies --- index.js | 35 ++-- lib/provider/einsAImmobilien.js | 8 +- lib/provider/immobilienDe.js | 8 +- lib/provider/immonet.js | 8 +- lib/provider/immoscout.js | 29 ++- lib/provider/immoswp.js | 8 +- lib/provider/immowelt.js | 8 +- lib/provider/kleinanzeigen.js | 10 +- lib/provider/neubauKompass.js | 6 +- lib/provider/wgGesucht.js | 8 +- .../demoCleanup-cron.js} | 8 +- lib/services/crons/listing-alive-cron.js | 13 ++ .../Tracker-Cron.js => crons/tracker-cron.js} | 2 +- .../immoscout/immoscout-web-translator.js | 12 ++ lib/services/listings/listingActiveService.js | 104 ++++++++++ lib/services/listings/listingActiveTester.js | 68 ++++++ lib/services/storage/listingsStorage.js | 34 ++- .../sql/2.active-flag-for-listings.js | 8 + lib/utils.js | 64 ++++-- package.json | 16 +- test/provider/utils.test.js | 16 +- yarn.lock | 193 ++++++++---------- 22 files changed, 487 insertions(+), 179 deletions(-) rename lib/services/{demoCleanup.js => crons/demoCleanup-cron.js} (72%) create mode 100644 lib/services/crons/listing-alive-cron.js rename lib/services/{tracking/Tracker-Cron.js => crons/tracker-cron.js} (88%) create mode 100644 lib/services/listings/listingActiveService.js create mode 100644 lib/services/listings/listingActiveTester.js create mode 100644 lib/services/storage/migrations/sql/2.active-flag-for-listings.js diff --git a/index.js b/index.js index 5dd4b7f..16179ab 100755 --- a/index.js +++ b/index.js @@ -1,16 +1,20 @@ import fs from 'fs'; import path from 'path'; -import { config } from './lib/utils.js'; +import { config, getProviders, refreshConfig } from './lib/utils.js'; import * as similarityCache from './lib/services/similarity-check/similarityCache.js'; import * as jobStorage from './lib/services/storage/jobStorage.js'; import FredyRuntime from './lib/FredyRuntime.js'; import { duringWorkingHoursOrNotSet } from './lib/utils.js'; import { runMigrations } from './lib/services/storage/migrations/migrate.js'; import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js'; -import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js'; -import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js'; +import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js'; +import { initTrackerCron } from './lib/services/crons/tracker-cron.js'; import logger from './lib/services/logger.js'; import { bus } from './lib/services/events/event-bus.js'; +import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js'; + +// Load configuration before any other startup steps +await refreshConfig(); // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath) const rawDir = config.sqlitepath || '/db'; @@ -23,8 +27,9 @@ if (!fs.existsSync(absDir)) { // Run DB migrations once at startup and block until finished await runMigrations(); -const providersPath = './lib/provider'; -const provider = fs.readdirSync(providersPath).filter((file) => file.endsWith('.js')); +// Load provider modules once at startup +const providers = await getProviders(); + //assuming interval is always in minutes const INTERVAL = config.interval * 60 * 1000; @@ -38,13 +43,11 @@ if (config.demoMode) { logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`); -const fetchedProvider = await Promise.all( - provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${providersPath}/${pro}`)), -); - ensureAdminUserExists(); ensureDemoUserExists(); await initTrackerCron(); +//do not wait for this to finish, let it run in the background +initActiveCheckerCron(); bus.on('jobs:runAll', () => { logger.debug('Running Fredy Job manually'); @@ -61,11 +64,17 @@ const execute = () => { .filter((job) => job.enabled) .forEach((job) => { job.provider - .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null) + .filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null) .forEach(async (prov) => { - const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id); - pro.init(prov, job.blacklist); - await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute(); + const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id); + matchedProvider.init(prov, job.blacklist); + await new FredyRuntime( + matchedProvider.config, + job.notificationAdapter, + prov.id, + job.id, + similarityCache, + ).execute(); }); }); } else { diff --git a/lib/provider/einsAImmobilien.js b/lib/provider/einsAImmobilien.js index 10fc319..eabe299 100755 --- a/lib/provider/einsAImmobilien.js +++ b/lib/provider/einsAImmobilien.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; function normalize(o) { @@ -29,8 +30,8 @@ function normalizePrice(price) { return result[0]; } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -49,6 +50,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immobilienDe.js b/lib/provider/immobilienDe.js index d614aa4..ba39412 100644 --- a/lib/provider/immobilienDe.js +++ b/lib/provider/immobilienDe.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -24,8 +25,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -46,6 +47,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immonet.js b/lib/provider/immonet.js index 5e1a2a2..2499e56 100755 --- a/lib/provider/immonet.js +++ b/lib/provider/immonet.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; function normalize(o) { @@ -11,8 +12,8 @@ function normalize(o) { return Object.assign(o, { id, address, price, size, title, link }); } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } const config = { @@ -31,6 +32,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immoscout.js b/lib/provider/immoscout.js index f1cf7ba..58e1e3b 100644 --- a/lib/provider/immoscout.js +++ b/lib/provider/immoscout.js @@ -35,8 +35,11 @@ * */ -import utils, { buildHash } from '../utils.js'; -import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js'; +import { buildHash, isOneOf } from '../utils.js'; +import { + convertImmoscoutListingToMobileListing, + convertWebToMobile, +} from '../services/immoscout/immoscout-web-translator.js'; import logger from '../services/logger.js'; let appliedBlackList = []; @@ -77,6 +80,25 @@ async function getListings(url) { }); } +async function isListingActive(link) { + const result = await fetch(convertImmoscoutListingToMobileListing(link), { + headers: { + 'User-Agent': 'ImmoScout_27.3_26.0_._', + }, + }); + + if (result.status === 200) { + return 1; + } + + if (result.status === 404) { + return 0; + } + + logger.warn('Unknown status for immoscout listing', link); + return -1; +} + function nullOrEmpty(val) { return val == null || val.length === 0; } @@ -87,7 +109,7 @@ function normalize(o) { return Object.assign(o, { id, title, address }); } function applyBlacklist(o) { - return !utils.isOneOf(o.title, appliedBlackList); + return !isOneOf(o.title, appliedBlackList); } const config = { url: null, @@ -104,6 +126,7 @@ const config = { normalize: normalize, filter: applyBlacklist, getListings: getListings, + activeTester: isListingActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immoswp.js b/lib/provider/immoswp.js index 7bb9548..5c911d1 100755 --- a/lib/provider/immoswp.js +++ b/lib/provider/immoswp.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -14,8 +15,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -35,6 +36,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immowelt.js b/lib/provider/immowelt.js index afdb057..924936e 100755 --- a/lib/provider/immowelt.js +++ b/lib/provider/immowelt.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -8,8 +9,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -30,6 +31,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index f21cd16..e322c5b 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; let appliedBlacklistedDistricts = []; @@ -11,10 +12,10 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); const isBlacklistedDistrict = - appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts); + appliedBlacklistedDistricts.length === 0 ? false : isOneOf(o.description, appliedBlacklistedDistricts); return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted; } @@ -36,6 +37,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const metaInformation = { name: 'Ebay Kleinanzeigen', diff --git a/lib/provider/neubauKompass.js b/lib/provider/neubauKompass.js index bdb2fc0..a6cf3cc 100755 --- a/lib/provider/neubauKompass.js +++ b/lib/provider/neubauKompass.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -15,7 +16,7 @@ function normalize(o) { } function applyBlacklist(o) { - return !utils.isOneOf(o.title, appliedBlackList); + return !isOneOf(o.title, appliedBlackList); } const config = { @@ -33,6 +34,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/wgGesucht.js b/lib/provider/wgGesucht.js index a287097..f0d441a 100755 --- a/lib/provider/wgGesucht.js +++ b/lib/provider/wgGesucht.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -10,8 +11,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return o.id != null && titleNotBlacklisted && descNotBlacklisted; } @@ -31,6 +32,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/services/demoCleanup.js b/lib/services/crons/demoCleanup-cron.js similarity index 72% rename from lib/services/demoCleanup.js rename to lib/services/crons/demoCleanup-cron.js index b21d383..79b5752 100644 --- a/lib/services/demoCleanup.js +++ b/lib/services/crons/demoCleanup-cron.js @@ -1,7 +1,7 @@ -import { removeJobsByUserId } from './storage/jobStorage.js'; -import { config } from '../utils.js'; -import { getUsers } from './storage/userStorage.js'; -import logger from './logger.js'; +import { removeJobsByUserId } from '../storage/jobStorage.js'; +import { config } from '../../utils.js'; +import { getUsers } from '../storage/userStorage.js'; +import logger from '../logger.js'; import cron from 'node-cron'; /** diff --git a/lib/services/crons/listing-alive-cron.js b/lib/services/crons/listing-alive-cron.js new file mode 100644 index 0000000..60dd5d9 --- /dev/null +++ b/lib/services/crons/listing-alive-cron.js @@ -0,0 +1,13 @@ +import cron from 'node-cron'; +import runActiveChecker from '../listings/listingActiveService.js'; + +async function runTask() { + await runActiveChecker(); +} + +export async function initActiveCheckerCron() { + //run directly on start + await runTask(); + // then every day at 1 am + cron.schedule('0 1 * * *', runTask); +} diff --git a/lib/services/tracking/Tracker-Cron.js b/lib/services/crons/tracker-cron.js similarity index 88% rename from lib/services/tracking/Tracker-Cron.js rename to lib/services/crons/tracker-cron.js index 9b89438..7cfe8cc 100644 --- a/lib/services/tracking/Tracker-Cron.js +++ b/lib/services/crons/tracker-cron.js @@ -1,6 +1,6 @@ import cron from 'node-cron'; import { config, inDevMode } from '../../utils.js'; -import { trackMainEvent } from './Tracker.js'; +import { trackMainEvent } from '../tracking/Tracker.js'; async function runTask() { //make sure to only send tracking events if the user gave us the green light and we are not in dev mode diff --git a/lib/services/immoscout/immoscout-web-translator.js b/lib/services/immoscout/immoscout-web-translator.js index 18a21e0..ec28957 100644 --- a/lib/services/immoscout/immoscout-web-translator.js +++ b/lib/services/immoscout/immoscout-web-translator.js @@ -60,6 +60,7 @@ https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC? */ import queryString from 'query-string'; +import { nullOrEmpty } from '../../utils.js'; const PARAM_NAME_MAP = { heatingtypes: 'heatingtypes', @@ -193,3 +194,14 @@ export function convertWebToMobile(webUrl) { return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`; } + +export function convertImmoscoutListingToMobileListing(url) { + if (nullOrEmpty(url)) { + return null; + } + + return url.replace( + /^https:\/\/www\.immobilienscout24\.de\/expose\//, + 'https://api.mobile.immobilienscout24.de/expose/', + ); +} diff --git a/lib/services/listings/listingActiveService.js b/lib/services/listings/listingActiveService.js new file mode 100644 index 0000000..23c56bb --- /dev/null +++ b/lib/services/listings/listingActiveService.js @@ -0,0 +1,104 @@ +import { deactivateListings, getActiveOrUnknownListings } from '../storage/listingsStorage.js'; +import { getProviders } from '../../utils.js'; +import logger from '../../services/logger.js'; + +/** + * Runs the active-listing checker: + * 1) Loads all listings with unknown or active status. + * 2) Resolves each listing's provider and calls its `activeTester(link)`. + * 3) Collects listings that are no longer active and deactivates them in one batch. + * + * Concurrency: network-bound checks are executed with a configurable concurrency limit. + * + * @param {object} [opts] + * @param {number} [opts.concurrency=8] Max number of parallel activeTester calls. + * @returns {Promise} + */ +export default async function runActiveChecker(opts = {}) { + const { concurrency = 4 } = opts; + + const listings = getActiveOrUnknownListings(); + if (!Array.isArray(listings) || listings.length === 0) { + logger.debug('No listings to check.'); + return; + } + + const providers = await getProviders(); + if (!Array.isArray(providers) || providers.length === 0) { + logger.warn('No providers available. Skipping active checks.'); + return; + } + + // Build a map for O(1) provider lookup by id + /** @type {Record} */ + const providerById = Object.create(null); + for (const p of providers) { + const id = p?.metaInformation?.id; + if (id) providerById[id] = p; + } + + // Small generic mapLimit to cap concurrency without extra deps + /** + * @template T, R + * @param {T[]} items + * @param {number} limit + * @param {(item: T, index: number) => Promise} worker + * @returns {Promise} + */ + async function mapLimit(items, limit, worker) { + const results = new Array(items.length); + let next = 0; + + async function runOne() { + while (next < items.length) { + const i = next++; + try { + results[i] = await worker(items[i], i); + } catch (err) { + results[i] = /** @type {any} */ (err); + } + } + } + + const runners = Array.from({ length: Math.min(limit, items.length) }, runOne); + await Promise.all(runners); + return results; + } + + /** @type {string[]} */ + const listingsSetToInactive = []; + + await mapLimit(listings, concurrency, async (listing) => { + const { provider: listingProviderId, link, id } = listing || {}; + + const matchedProvider = providerById[listingProviderId]; + if (!matchedProvider) { + logger.warn('Could not find matching provider for', listingProviderId); + return; + } + const tester = matchedProvider?.config?.activeTester; + if (typeof tester !== 'function') { + logger.warn('No activeTester configured for', listingProviderId); + return; + } + + // Contract: activeTester(link) returns 1 if active, 0 if inactive + let result; + try { + result = await tester(link); + } catch { + result = -1; + } + + if (result === 0 && id) { + listingsSetToInactive.push(id); + } + }); + + if (listingsSetToInactive.length > 0) { + logger.info(`Setting ${listingsSetToInactive.length} listings to inactive.`); + deactivateListings(listingsSetToInactive); + } else { + logger.debug('No listings need to be set inactive.'); + } +} diff --git a/lib/services/listings/listingActiveTester.js b/lib/services/listings/listingActiveTester.js new file mode 100644 index 0000000..148c925 --- /dev/null +++ b/lib/services/listings/listingActiveTester.js @@ -0,0 +1,68 @@ +import fetch from 'node-fetch'; +import { randomBetween, sleep } from '../../utils.js'; + +const maxAttempts = 3; + +/** + * Check if a listing is still active with up to 3 attempts and exponential backoff. + * Backoff waits are capped and the last wait is at most 2000 ms. + * + * Rules: + * - HTTP 200 => return 1 + * - HTTP 401/403 => return -1 (most certainly detected as a bot) + * - HTTP 404 => return 0 + * - Other statuses or network errors => retry until attempts are exhausted + * + * @returns {Promise} 1 if active, o if not active and -1 if detected as bot + */ +export default async function checkIfListingIsActive(link) { + await sleep(randomBetween(50, 100)); + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const res = await fetch(link, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', + 'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8', + }, + }); + + if (res.status === 200) { + return 1; + } + if (res.status === 401) return -1; + if (res.status === 403) return -1; + if (res.status === 404) return 0; + + // For any other status, only retry if attempts remain + if (attempt < maxAttempts) { + await sleep(backoffDelay(attempt)); + continue; + } + + return 0; + } catch { + // Network error: retry if attempts remain + if (attempt < maxAttempts) { + await sleep(backoffDelay(attempt)); + continue; + } + return 0; + } + } + + return 0; +} + +/** + * Exponential backoff delay with cap. + * attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap) + * @param {number} attempt 1-based attempt index + * @returns {number} delay in ms + */ +function backoffDelay(attempt) { + const base = 500; + const cap = 2000; + return Math.min(base * 2 ** (attempt - 1), cap); +} diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index d1b0075..082767b 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -53,6 +53,36 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => { ).map((r) => r.hash); }; +/** + * Return a list of listing that either are active or have an unknown status + * to constantly check if they are still online + * + * @returns {string[]} Array of listings + */ +export const getActiveOrUnknownListings = () => { + return SqliteConnection.query( + `SELECT * + FROM listings + WHERE is_active is null OR is_active = 1 ORDER BY provider`, + ); +}; + +/** + * Deactivates listings by setting is_active = 0 for all matching IDs. + * + * @param {string[]} ids - Array of listing IDs to deactivate. + * @returns {object[]} Result of the SQLite query execution. + */ +export const deactivateListings = (ids) => { + const placeholders = ids.map(() => '?').join(','); + return SqliteConnection.execute( + `UPDATE listings + SET is_active = 0 + WHERE id IN (${placeholders})`, + ids, + ); +}; + /** * Persist a batch of scraped listings for a given job and provider. * @@ -86,9 +116,9 @@ export const storeListings = (jobId, providerId, listings) => { SqliteConnection.withTransaction((db) => { const stmt = db.prepare( `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, - link, created_at) + link, created_at, is_active) VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link, - @created_at) + @created_at, 1) ON CONFLICT(job_id, hash) DO NOTHING`, ); diff --git a/lib/services/storage/migrations/sql/2.active-flag-for-listings.js b/lib/services/storage/migrations/sql/2.active-flag-for-listings.js new file mode 100644 index 0000000..521c467 --- /dev/null +++ b/lib/services/storage/migrations/sql/2.active-flag-for-listings.js @@ -0,0 +1,8 @@ +// Migration: there needs to be a unique index on job_id and hash as only +// this makes the listing indeed unique + +export function up(db) { + db.exec(` + ALTER TABLE listings ADD COLUMN is_active INTEGER DEFAULT 1; + `); +} diff --git a/lib/utils.js b/lib/utils.js index d752339..7f9c3bd 100755 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,5 +1,6 @@ import { dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { readFile } from 'fs/promises'; import { createHash } from 'crypto'; import { DEFAULT_CONFIG } from './defaultConfig.js'; @@ -11,6 +12,26 @@ const RE_GT = />/g; const RE_WEBP = /\/format\/webp/gi; const RE_EXT = /\.(jpe?g|png|gif)(\?.*)?$/i; const HTTPS_PREFIX = 'https://'; +const providersDirectoryPath = `${getDirName()}/provider`; + +/** + * Lazily load all provider modules from the provider directory. + * Caches the resolved array to avoid re-importing on subsequent calls. + * + * @returns {Promise} A list of loaded provider modules. + */ +let cachedProvidersPromise = null; + +export function getProviders() { + if (!cachedProvidersPromise) { + /** @type {string[]} */ + const providerFileNames = fs.readdirSync(providersDirectoryPath).filter((fileName) => fileName.endsWith('.js')); + cachedProvidersPromise = Promise.all( + providerFileNames.map((fileName) => import(pathToFileURL(path.join(providersDirectoryPath, fileName)).href)), + ); + } + return cachedProvidersPromise; +} /** * Safely stringify a value to JSON for storage. @@ -21,7 +42,7 @@ const HTTPS_PREFIX = 'https://'; * @param {T} v - Any JSON-serializable value. * @returns {string|null} JSON string or null. */ -export const toJson = (v) => (v == null ? null : JSON.stringify(v)); +const toJson = (v) => (v == null ? null : JSON.stringify(v)); /** * Safely parse JSON text coming from storage. @@ -33,7 +54,7 @@ export const toJson = (v) => (v == null ? null : JSON.stringify(v)); * @param {T} fallback - Value to return when txt is null/invalid. * @returns {T} Parsed value or fallback. */ -export const fromJson = (txt, fallback) => { +const fromJson = (txt, fallback) => { if (txt == null) return fallback; try { return JSON.parse(txt); @@ -213,23 +234,40 @@ async function getPackageVersion() { return 'N/A'; } +/** + * Sleep helper + * @param {number} ms milliseconds to wait + * @returns {Promise} + */ +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * returns a random into between start and end + * @param a start int + * @param b max int + * @returns {*} + */ +function randomBetween(a, b) { + return Math.floor(Math.random() * (b - a + 1)) + a; +} + +// Call refreshConfig() from the application entrypoint during startup to populate config. await refreshConfig(); -export { isOneOf }; -export { normalizeImageUrl }; -export { inDevMode }; -export { nullOrEmpty }; -export { duringWorkingHoursOrNotSet }; -export { getDirName }; -export { config }; -export { buildHash }; -export { getPackageVersion }; -export default { +export { isOneOf, + normalizeImageUrl, + inDevMode, nullOrEmpty, duringWorkingHoursOrNotSet, getDirName, + sleep, + randomBetween, config, + buildHash, + getPackageVersion, toJson, fromJson, }; diff --git a/package.json b/package.json index 02143cb..386ebc7 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fredy", - "version": "12.1.4", + "version": "12.1.5", "description": "[F]ind [R]eal [E]states [d]amn eas[y].", "scripts": { "prepare": "husky", @@ -58,12 +58,12 @@ "dependencies": { "@douyinfe/semi-icons": "^2.86.0", "@douyinfe/semi-ui": "2.86.0", - "@sendgrid/mail": "8.1.5", - "@visactor/react-vchart": "^2.0.4", - "@visactor/vchart": "^2.0.4", + "@sendgrid/mail": "8.1.6", + "@visactor/react-vchart": "^2.0.5", + "@visactor/vchart": "^2.0.5", "@visactor/vchart-semi-theme": "^1.12.2", "@vitejs/plugin-react": "5.0.3", - "better-sqlite3": "^12.2.0", + "better-sqlite3": "^12.3.0", "body-parser": "2.2.0", "cheerio": "^1.1.2", "cookie-session": "2.1.1", @@ -87,7 +87,7 @@ "restana": "5.1.0", "serve-static": "2.2.0", "slack": "11.0.2", - "vite": "7.1.6", + "vite": "7.1.7", "x-var": "^3.0.1", "zustand": "^5.0.8" }, @@ -97,14 +97,14 @@ "@babel/preset-env": "7.28.3", "@babel/preset-react": "7.27.1", "chai": "6.0.1", - "eslint": "9.35.0", + "eslint": "9.36.0", "eslint-config-prettier": "10.1.8", "eslint-plugin-react": "7.37.5", "esmock": "2.7.3", "history": "5.3.0", "husky": "9.1.7", "less": "4.4.1", - "lint-staged": "16.1.6", + "lint-staged": "16.2.0", "mocha": "11.7.2", "nodemon": "^3.1.10", "prettier": "3.6.2" diff --git a/test/provider/utils.test.js b/test/provider/utils.test.js index f993daf..8dc9fb3 100644 --- a/test/provider/utils.test.js +++ b/test/provider/utils.test.js @@ -1,4 +1,4 @@ -import utils from '../../lib/utils.js'; +import { isOneOf, duringWorkingHoursOrNotSet } from '../../lib/utils.js'; import assert from 'assert'; import { expect } from 'chai'; @@ -11,27 +11,27 @@ const fakeWorkingHoursConfig = (from, to) => ({ describe('utils', () => { describe('#isOneOf()', () => { it('should be false', () => { - assert.equal(utils.isOneOf('bla', ['blub']), false); + assert.equal(isOneOf('bla', ['blub']), false); }); it('should be true', () => { - assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true); + assert.equal(isOneOf('bla blub blubber', ['bla']), true); }); }); describe('#duringWorkingHoursOrNotSet()', () => { it('should be false', () => { - expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false; + expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false; }); it('should be true', () => { - expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true; + expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true; }); it('should be true if nothing set', () => { - expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true; + expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true; }); it('should be true if only to is set', () => { - expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true; + expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true; }); it('should be true if only from is set', () => { - expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true; + expect(duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true; }); }); }); diff --git a/yarn.lock b/yarn.lock index 7c89705..df4458f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1203,10 +1203,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.35.0": - version "9.35.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.35.0.tgz#ffbc7e13cf1204db18552e9cd9d4a8e17c692d07" - integrity sha512-30iXE9whjlILfWobBkNerJo+TXYsgVM5ERQwMcMKCHckHflCmf7wXDAHlARoWnh0s1U72WqlbeyE7iAcCzuCPw== +"@eslint/js@9.36.0": + version "9.36.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.36.0.tgz#b1a3893dd6ce2defed5fd49de805ba40368e8fef" + integrity sha512-uhCbYtYynH30iZErszX78U+nR3pJU3RHGQ57NXy5QupD4SBVwDeU8TNBy+MjMngc1UyIW9noKqsRqfjQTBU2dw== "@eslint/object-schema@^2.1.6": version "2.1.6" @@ -1548,10 +1548,10 @@ dependencies: deepmerge "^4.2.2" -"@sendgrid/mail@8.1.5": - version "8.1.5" - resolved "https://registry.yarnpkg.com/@sendgrid/mail/-/mail-8.1.5.tgz#995ef96aaf4664d2f059ec6ca38f79f724d350f2" - integrity sha512-W+YuMnkVs4+HA/bgfto4VHKcPKLc7NiZ50/NH2pzO6UHCCFuq8/GNB98YJlLEr/ESDyzAaDr7lVE7hoBwFTT3Q== +"@sendgrid/mail@8.1.6": + version "8.1.6" + resolved "https://registry.yarnpkg.com/@sendgrid/mail/-/mail-8.1.6.tgz#9c253c13d49867fdb6f7df1360643825236eef22" + integrity sha512-/ZqxUvKeEztU9drOoPC/8opEPOk+jLlB2q4+xpx6HVLq6aFu3pMpalkTpAQz8XfRfpLp8O25bh6pGPcHDCYpqg== dependencies: "@sendgrid/client" "^8.1.5" "@sendgrid/helpers" "^8.0.0" @@ -1729,24 +1729,24 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.0.tgz#d06bbb384ebcf6c505fde1c3d0ed4ddffe0aaff8" integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g== -"@visactor/react-vchart@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@visactor/react-vchart/-/react-vchart-2.0.4.tgz#221760d3c9707fcee9e94b3b0fd0371540d40db0" - integrity sha512-dN0VHEXMF1QTA9JAaV1kZYxajxwwPBpMhLB1vXgY9u41prDFYyboQ7atwweyBB/xSdRdsuQgzYU/SSM/R2gNeg== +"@visactor/react-vchart@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@visactor/react-vchart/-/react-vchart-2.0.5.tgz#1eb3339b662f623c08cf20f57c2507760c784468" + integrity sha512-D3dAPASde1zuZiorx32jkRe9cMuc9PO3IVurw0Sm/XBzrdQE2MnoLONfM2ktT/BJQggBZaHE6+n8inGE24JyJg== dependencies: - "@visactor/vchart" "2.0.4" - "@visactor/vchart-extension" "2.0.4" + "@visactor/vchart" "2.0.5" + "@visactor/vchart-extension" "2.0.5" "@visactor/vrender-core" "1.0.13" "@visactor/vrender-kits" "1.0.13" "@visactor/vutils" "~1.0.6" react-is "^18.2.0" -"@visactor/vchart-extension@2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@visactor/vchart-extension/-/vchart-extension-2.0.4.tgz#8ac5e138bc410d9e9b23bb3e60547f01df48bac9" - integrity sha512-KmoeI7nxpfu8vGnn86O9szjoWTtvAomBtUwdtg+cNYkX/EGxZ4LUZLe0lELSpUecRk1aqZxzdeBSFB1wQpNYRw== +"@visactor/vchart-extension@2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@visactor/vchart-extension/-/vchart-extension-2.0.5.tgz#3c023ebd56bc26531f20c2ad147e45d1fcba67ef" + integrity sha512-GG5cwtJ3wv4/DUM4/RVF7qi6WXRZyDRIv+U0WgWCYAdANINW95egJ3P+NHdcdLhA7VEdAXPde6XFSWOawcK4oQ== dependencies: - "@visactor/vchart" "2.0.4" + "@visactor/vchart" "2.0.5" "@visactor/vdataset" "~1.0.6" "@visactor/vlayouts" "~1.0.6" "@visactor/vrender-animate" "1.0.13" @@ -1767,10 +1767,10 @@ resolved "https://registry.yarnpkg.com/@visactor/vchart-theme-utils/-/vchart-theme-utils-1.12.2.tgz#bad0035e79dabbe80890bbd6196668551a12c874" integrity sha512-PkgSAivtUZukCWVUGCXxKcbTzI/oMj1Ky22VYcVs/KM4VFmmCywU2xjBBe1du0LUey6CAKB7bMlj5bL2jctG0A== -"@visactor/vchart@2.0.4", "@visactor/vchart@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@visactor/vchart/-/vchart-2.0.4.tgz#36770240ae6ffd84fa285b7610192f2e06a56299" - integrity sha512-/NWBQFYd5A52I8Bkp+iod2LAhBo4cQcxt+xazrmJ/5L17Gk/LdUqCRpnF5dk3XncHb4ls+SRNGkH4kf0rNH2Mg== +"@visactor/vchart@2.0.5", "@visactor/vchart@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@visactor/vchart/-/vchart-2.0.5.tgz#a7041a1fe6df5125ca02ac55946b0211f4e649ed" + integrity sha512-7emhEFGEhUZC8n/PkscVQeJn/yd4757wrta1avMHUKBVY7x9qEWYSFypXT2LJTxjTePB//dqZYE/aPy/plGWNQ== dependencies: "@visactor/vdataset" "~1.0.6" "@visactor/vlayouts" "~1.0.6" @@ -1780,7 +1780,7 @@ "@visactor/vrender-kits" "1.0.13" "@visactor/vscale" "~1.0.6" "@visactor/vutils" "~1.0.6" - "@visactor/vutils-extension" "2.0.4" + "@visactor/vutils-extension" "2.0.5" "@visactor/vdataset@~1.0.6": version "1.0.9" @@ -1869,10 +1869,10 @@ dependencies: "@visactor/vutils" "1.0.9" -"@visactor/vutils-extension@2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@visactor/vutils-extension/-/vutils-extension-2.0.4.tgz#a369192d0ca5dd9748a21a5f1f6eb3ea094cac6c" - integrity sha512-Q0nDVTCLeCbAi8AAj8wAZfzfZDDsYF7xXhuLjjGPrPTuItPG/fHuw/rw6yDFvdhb4XGaPwv0MaUYNPFoOl60GQ== +"@visactor/vutils-extension@2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@visactor/vutils-extension/-/vutils-extension-2.0.5.tgz#7c713c6c2bdced9c7ab599d5444b37c80ce8f8c7" + integrity sha512-qQpaANT+AtOQoQAN64qhQQXqhOo9Fn5t+hmih0pFxIye+61yEj3xUSM2GxQF6ubjqCI6DvRG0DaVw0rdcoqbGg== dependencies: "@visactor/vdataset" "~1.0.6" "@visactor/vutils" "~1.0.6" @@ -1966,7 +1966,7 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" -ansi-styles@^6.0.0, ansi-styles@^6.1.0, ansi-styles@^6.2.1: +ansi-styles@^6.1.0, ansi-styles@^6.2.1: version "6.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== @@ -2197,10 +2197,10 @@ basic-ftp@^5.0.2: resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.0.5.tgz#14a474f5fffecca1f4f406f1c26b18f800225ac0" integrity sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg== -better-sqlite3@^12.2.0: - version "12.2.0" - resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.2.0.tgz#de7c3466074f2d1a5d260f510647e822e42684d2" - integrity sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ== +better-sqlite3@^12.3.0: + version "12.3.0" + resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.3.0.tgz#999817506ed9d985604ae053b5e5fe3c8a052bb1" + integrity sha512-FFf+rsghyvXQIPV/6PDUj05EsuZA1b0drGLzNgtrELkXnJKUH6NNM2h7Ce7dkA6vvPOM4SOoUIDGRPy3yRKmqw== dependencies: bindings "^1.5.0" prebuild-install "^7.1.1" @@ -2375,11 +2375,6 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^5.6.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.6.0.tgz#a1a8d294ea3526dbb77660f12649a08490e33ab8" - integrity sha512-46QrSQFyVSEyYAgQ22hQ+zDa60YHA4fBstHmtSApj1Y5vKtG27fWowW03jCk5KcbXEWPZUIR894aARCA/G1kfQ== - character-entities-html4@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-2.1.0.tgz#1f1adb940c971a4b22ba39ddca6b618dc6e56b2b" @@ -2476,13 +2471,13 @@ cli-cursor@^5.0.0: dependencies: restore-cursor "^5.0.0" -cli-truncate@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-4.0.0.tgz#6cc28a2924fee9e25ce91e973db56c7066e6172a" - integrity sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA== +cli-truncate@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-5.1.0.tgz#bb12607a62f0e4bb91a54aa4653b23347900bb55" + integrity sha512-7JDGG+4Zp0CsknDCedl0DYdaeOhc46QNpXi3NLQblkZpXXgA6LncLDUUyvrjSvZeF3VRQa+KiMGomazQrC1V8g== dependencies: - slice-ansi "^5.0.0" - string-width "^7.0.0" + slice-ansi "^7.1.0" + string-width "^8.0.0" cliui@^8.0.1: version "8.0.1" @@ -2543,16 +2538,16 @@ comma-separated-tokens@^2.0.0: resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== +commander@14.0.1: + version "14.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.1.tgz#2f9225c19e6ebd0dc4404dd45821b2caa17ea09b" + integrity sha512-2JkV3gUZUVrbNA+1sjBOYLsMZ5cEEl8GTFP2a4AVz5hvasAMCQ1D2l2le/cX+pV4N6ZU17zjUahLpIXRrnWL8A== + commander@2: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== -commander@^14.0.0: - version "14.0.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.0.tgz#f244fc74a92343514e56229f16ef5c5e22ced5e9" - integrity sha512-2uM9rYjPvyq39NwLRqaiLtWHyDC1FvryJDa2ATTVims5YAS4PupsEQsDvP14FqhFr0P49CYDugi59xaxJlTXRA== - compute-scroll-into-view@^1.0.20: version "1.0.20" resolved "https://registry.yarnpkg.com/compute-scroll-into-view/-/compute-scroll-into-view-1.0.20.tgz#1768b5522d1172754f5d0c9b02de3af6be506a43" @@ -3281,10 +3276,10 @@ eslint-visitor-keys@^4.2.1: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz#4cfea60fe7dd0ad8e816e1ed026c1d5251b512c1" integrity sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ== -eslint@9.35.0: - version "9.35.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.35.0.tgz#7a89054b7b9ee1dfd1b62035d8ce75547773f47e" - integrity sha512-QePbBFMJFjgmlE+cXAlbHZbHpdFVS2E/6vzCy7aKlebddvl1vadiC4JFV5u/wqTkNUwEV8WrQi257jf5f06hrg== +eslint@9.36.0: + version "9.36.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.36.0.tgz#9cc5cbbfb9c01070425d9bfed81b4e79a1c09088" + integrity sha512-hB4FIzXovouYzwzECDcUkJ4OcfOEkXTv2zRY6B9bkwjx/cprAq0uvm1nl7zvQ0/TsUk0zQiN4uPfJpB9m+rPMQ== dependencies: "@eslint-community/eslint-utils" "^4.8.0" "@eslint-community/regexpp" "^4.12.1" @@ -3292,7 +3287,7 @@ eslint@9.35.0: "@eslint/config-helpers" "^0.3.1" "@eslint/core" "^0.15.2" "@eslint/eslintrc" "^3.3.1" - "@eslint/js" "9.35.0" + "@eslint/js" "9.36.0" "@eslint/plugin-kit" "^0.3.5" "@humanfs/node" "^0.16.6" "@humanwhocodes/module-importer" "^1.0.1" @@ -3721,6 +3716,11 @@ get-east-asian-width@^1.0.0: resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz#21b4071ee58ed04ee0db653371b55b4299875389" integrity sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ== +get-east-asian-width@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz#9bc4caa131702b4b61729cb7e42735bc550c9ee6" + integrity sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q== + get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.2.7, get-intrinsic@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" @@ -4222,11 +4222,6 @@ is-fullwidth-code-point@^3.0.0: resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== -is-fullwidth-code-point@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz#fae3167c729e7463f8461ce512b080a49268aa88" - integrity sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ== - is-fullwidth-code-point@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-5.0.0.tgz#9609efced7c2f97da7b60145ef481c787c7ba704" @@ -4559,38 +4554,30 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -lilconfig@^3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-3.1.3.tgz#a1bcfd6257f9585bf5ae14ceeebb7b559025e4c4" - integrity sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw== - lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== -lint-staged@16.1.6: - version "16.1.6" - resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-16.1.6.tgz#b0830df339a71f4207979a47c7be8ab0f38543ad" - integrity sha512-U4kuulU3CKIytlkLlaHcGgKscNfJPNTiDF2avIUGFCv7K95/DCYQ7Ra62ydeRWmgQGg9zJYw2dzdbztwJlqrow== +lint-staged@16.2.0: + version "16.2.0" + resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-16.2.0.tgz#ea7157bf007bdb50d2bb0559bc91c8e77d71c84b" + integrity sha512-spdYSOCQ2MdZ9CM1/bu/kDmaYGsrpNOeu1InFFV8uhv14x6YIubGxbCpSmGILFoxkiheNQPDXSg5Sbb5ZuVnug== dependencies: - chalk "^5.6.0" - commander "^14.0.0" - debug "^4.4.1" - lilconfig "^3.1.3" - listr2 "^9.0.3" - micromatch "^4.0.8" - nano-spawn "^1.0.2" - pidtree "^0.6.0" - string-argv "^0.3.2" - yaml "^2.8.1" + commander "14.0.1" + listr2 "9.0.4" + micromatch "4.0.8" + nano-spawn "1.0.3" + pidtree "0.6.0" + string-argv "0.3.2" + yaml "2.8.1" -listr2@^9.0.3: - version "9.0.3" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-9.0.3.tgz#5181284019e1d577dc2d705ca6d3a148cf15adf3" - integrity sha512-0aeh5HHHgmq1KRdMMDHfhMWQmIT/m7nRDTlxlFqni2Sp0had9baqsjJRvDGdlvgd6NmPE0nPloOipiQJGFtTHQ== +listr2@9.0.4: + version "9.0.4" + resolved "https://registry.yarnpkg.com/listr2/-/listr2-9.0.4.tgz#2916e633ae6e09d1a3f981172937ac1c5a8fa64f" + integrity sha512-1wd/kpAdKRLwv7/3OKC8zZ5U8e/fajCfWMxacUvB79S5nLrYGPtUI/8chMQhn3LQjsRVErTb9i1ECAwW0ZIHnQ== dependencies: - cli-truncate "^4.0.0" + cli-truncate "^5.0.0" colorette "^2.0.20" eventemitter3 "^5.0.1" log-update "^6.1.0" @@ -5284,7 +5271,7 @@ micromark@^4.0.0: micromark-util-symbol "^2.0.0" micromark-util-types "^2.0.0" -micromatch@^4.0.8: +micromatch@4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== @@ -5414,10 +5401,10 @@ ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -nano-spawn@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/nano-spawn/-/nano-spawn-1.0.2.tgz#9853795681f0e96ef6f39104c2e4347b6ba79bf6" - integrity sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg== +nano-spawn@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/nano-spawn/-/nano-spawn-1.0.3.tgz#ef8d89a275eebc8657e67b95fc312a6527a05b8d" + integrity sha512-jtpsQDetTnvS2Ts1fiRdci5rx0VYws5jGyC+4IYOTnIQ/wwdf6JdomlHBwqC3bJYOvaKu0C2GSZ1A60anrYpaA== nanoid@5.1.5: version "5.1.5" @@ -5830,7 +5817,7 @@ picomatch@^4.0.3: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.3.tgz#796c76136d1eead715db1e7bad785dedd695a042" integrity sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q== -pidtree@^0.6.0: +pidtree@0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/pidtree/-/pidtree-0.6.0.tgz#90ad7b6d42d5841e69e0a2419ef38f8883aa057c" integrity sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g== @@ -6755,14 +6742,6 @@ slack@11.0.2: dependencies: tiny-json-http "^7.0.2" -slice-ansi@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-5.0.0.tgz#b73063c57aa96f9cd881654b15294d95d285c42a" - integrity sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ== - dependencies: - ansi-styles "^6.0.0" - is-fullwidth-code-point "^4.0.0" - slice-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-7.1.0.tgz#cd6b4655e298a8d1bdeb04250a433094b347b9a9" @@ -6856,7 +6835,7 @@ streamx@^2.15.0, streamx@^2.21.0: optionalDependencies: bare-events "^2.2.0" -string-argv@^0.3.2: +string-argv@0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== @@ -6897,6 +6876,14 @@ string-width@^7.0.0: get-east-asian-width "^1.0.0" strip-ansi "^7.1.0" +string-width@^8.0.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-8.1.0.tgz#9e9fb305174947cf45c30529414b5da916e9e8d1" + integrity sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg== + dependencies: + get-east-asian-width "^1.3.0" + strip-ansi "^7.1.0" + string.prototype.matchall@^4.0.12: version "4.0.12" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz#6c88740e49ad4956b1332a911e949583a275d4c0" @@ -7421,10 +7408,10 @@ vfile@^6.0.0: "@types/unist" "^3.0.0" vfile-message "^4.0.0" -vite@7.1.6: - version "7.1.6" - resolved "https://registry.yarnpkg.com/vite/-/vite-7.1.6.tgz#336806d29983135677f498a05efb0fd46c5eef2d" - integrity sha512-SRYIB8t/isTwNn8vMB3MR6E+EQZM/WG1aKmmIUCfDXfVvKfc20ZpamngWHKzAmmu9ppsgxsg4b2I7c90JZudIQ== +vite@7.1.7: + version "7.1.7" + resolved "https://registry.yarnpkg.com/vite/-/vite-7.1.7.tgz#ed3f9f06e21d6574fe1ad425f6b0912d027ffc13" + integrity sha512-VbA8ScMvAISJNJVbRDTJdCwqQoAareR/wutevKanhR2/1EkoXVZVkkORaYm/tNVCjP/UDTKtcw3bAkwOUdedmA== dependencies: esbuild "^0.25.0" fdir "^6.5.0" @@ -7596,7 +7583,7 @@ yallist@^3.0.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yaml@^2.8.1: +yaml@2.8.1: version "2.8.1" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.1.tgz#1870aa02b631f7e8328b93f8bc574fac5d6c4d79" integrity sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==