diff --git a/index.js b/index.js index 5dd4b7f..16179ab 100755 --- a/index.js +++ b/index.js @@ -1,16 +1,20 @@ import fs from 'fs'; import path from 'path'; -import { config } from './lib/utils.js'; +import { config, getProviders, refreshConfig } from './lib/utils.js'; import * as similarityCache from './lib/services/similarity-check/similarityCache.js'; import * as jobStorage from './lib/services/storage/jobStorage.js'; import FredyRuntime from './lib/FredyRuntime.js'; import { duringWorkingHoursOrNotSet } from './lib/utils.js'; import { runMigrations } from './lib/services/storage/migrations/migrate.js'; import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js'; -import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js'; -import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js'; +import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js'; +import { initTrackerCron } from './lib/services/crons/tracker-cron.js'; import logger from './lib/services/logger.js'; import { bus } from './lib/services/events/event-bus.js'; +import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js'; + +// Load configuration before any other startup steps +await refreshConfig(); // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath) const rawDir = config.sqlitepath || '/db'; @@ -23,8 +27,9 @@ if (!fs.existsSync(absDir)) { // Run DB migrations once at startup and block until finished await runMigrations(); -const providersPath = './lib/provider'; -const provider = fs.readdirSync(providersPath).filter((file) => file.endsWith('.js')); +// Load provider modules once at startup +const providers = await getProviders(); + //assuming interval is always in minutes const INTERVAL = config.interval * 60 * 1000; @@ -38,13 +43,11 @@ if (config.demoMode) { logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`); -const fetchedProvider = await Promise.all( - provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${providersPath}/${pro}`)), -); - ensureAdminUserExists(); ensureDemoUserExists(); await initTrackerCron(); +//do not wait for this to finish, let it run in the background +initActiveCheckerCron(); bus.on('jobs:runAll', () => { logger.debug('Running Fredy Job manually'); @@ -61,11 +64,17 @@ const execute = () => { .filter((job) => job.enabled) .forEach((job) => { job.provider - .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null) + .filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null) .forEach(async (prov) => { - const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id); - pro.init(prov, job.blacklist); - await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute(); + const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id); + matchedProvider.init(prov, job.blacklist); + await new FredyRuntime( + matchedProvider.config, + job.notificationAdapter, + prov.id, + job.id, + similarityCache, + ).execute(); }); }); } else { diff --git a/lib/provider/einsAImmobilien.js b/lib/provider/einsAImmobilien.js index 10fc319..eabe299 100755 --- a/lib/provider/einsAImmobilien.js +++ b/lib/provider/einsAImmobilien.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; function normalize(o) { @@ -29,8 +30,8 @@ function normalizePrice(price) { return result[0]; } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -49,6 +50,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immobilienDe.js b/lib/provider/immobilienDe.js index d614aa4..ba39412 100644 --- a/lib/provider/immobilienDe.js +++ b/lib/provider/immobilienDe.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -24,8 +25,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -46,6 +47,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immonet.js b/lib/provider/immonet.js index 5e1a2a2..2499e56 100755 --- a/lib/provider/immonet.js +++ b/lib/provider/immonet.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; function normalize(o) { @@ -11,8 +12,8 @@ function normalize(o) { return Object.assign(o, { id, address, price, size, title, link }); } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } const config = { @@ -31,6 +32,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immoscout.js b/lib/provider/immoscout.js index f1cf7ba..58e1e3b 100644 --- a/lib/provider/immoscout.js +++ b/lib/provider/immoscout.js @@ -35,8 +35,11 @@ * */ -import utils, { buildHash } from '../utils.js'; -import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js'; +import { buildHash, isOneOf } from '../utils.js'; +import { + convertImmoscoutListingToMobileListing, + convertWebToMobile, +} from '../services/immoscout/immoscout-web-translator.js'; import logger from '../services/logger.js'; let appliedBlackList = []; @@ -77,6 +80,25 @@ async function getListings(url) { }); } +async function isListingActive(link) { + const result = await fetch(convertImmoscoutListingToMobileListing(link), { + headers: { + 'User-Agent': 'ImmoScout_27.3_26.0_._', + }, + }); + + if (result.status === 200) { + return 1; + } + + if (result.status === 404) { + return 0; + } + + logger.warn('Unknown status for immoscout listing', link); + return -1; +} + function nullOrEmpty(val) { return val == null || val.length === 0; } @@ -87,7 +109,7 @@ function normalize(o) { return Object.assign(o, { id, title, address }); } function applyBlacklist(o) { - return !utils.isOneOf(o.title, appliedBlackList); + return !isOneOf(o.title, appliedBlackList); } const config = { url: null, @@ -104,6 +126,7 @@ const config = { normalize: normalize, filter: applyBlacklist, getListings: getListings, + activeTester: isListingActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immoswp.js b/lib/provider/immoswp.js index 7bb9548..5c911d1 100755 --- a/lib/provider/immoswp.js +++ b/lib/provider/immoswp.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -14,8 +15,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -35,6 +36,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/immowelt.js b/lib/provider/immowelt.js index afdb057..924936e 100755 --- a/lib/provider/immowelt.js +++ b/lib/provider/immowelt.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -8,8 +9,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return titleNotBlacklisted && descNotBlacklisted; } @@ -30,6 +31,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index f21cd16..e322c5b 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { buildHash, isOneOf } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; let appliedBlacklistedDistricts = []; @@ -11,10 +12,10 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); const isBlacklistedDistrict = - appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts); + appliedBlacklistedDistricts.length === 0 ? false : isOneOf(o.description, appliedBlacklistedDistricts); return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted; } @@ -36,6 +37,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const metaInformation = { name: 'Ebay Kleinanzeigen', diff --git a/lib/provider/neubauKompass.js b/lib/provider/neubauKompass.js index bdb2fc0..a6cf3cc 100755 --- a/lib/provider/neubauKompass.js +++ b/lib/provider/neubauKompass.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -15,7 +16,7 @@ function normalize(o) { } function applyBlacklist(o) { - return !utils.isOneOf(o.title, appliedBlackList); + return !isOneOf(o.title, appliedBlackList); } const config = { @@ -33,6 +34,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/provider/wgGesucht.js b/lib/provider/wgGesucht.js index a287097..f0d441a 100755 --- a/lib/provider/wgGesucht.js +++ b/lib/provider/wgGesucht.js @@ -1,4 +1,5 @@ -import utils, { buildHash } from '../utils.js'; +import { isOneOf, buildHash } from '../utils.js'; +import checkIfListingIsActive from '../services/listings/listingActiveTester.js'; let appliedBlackList = []; @@ -10,8 +11,8 @@ function normalize(o) { } function applyBlacklist(o) { - const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList); - const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList); + const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList); + const descNotBlacklisted = !isOneOf(o.description, appliedBlackList); return o.id != null && titleNotBlacklisted && descNotBlacklisted; } @@ -31,6 +32,7 @@ const config = { }, normalize: normalize, filter: applyBlacklist, + activeTester: checkIfListingIsActive, }; export const init = (sourceConfig, blacklist) => { config.enabled = sourceConfig.enabled; diff --git a/lib/services/demoCleanup.js b/lib/services/crons/demoCleanup-cron.js similarity index 72% rename from lib/services/demoCleanup.js rename to lib/services/crons/demoCleanup-cron.js index b21d383..79b5752 100644 --- a/lib/services/demoCleanup.js +++ b/lib/services/crons/demoCleanup-cron.js @@ -1,7 +1,7 @@ -import { removeJobsByUserId } from './storage/jobStorage.js'; -import { config } from '../utils.js'; -import { getUsers } from './storage/userStorage.js'; -import logger from './logger.js'; +import { removeJobsByUserId } from '../storage/jobStorage.js'; +import { config } from '../../utils.js'; +import { getUsers } from '../storage/userStorage.js'; +import logger from '../logger.js'; import cron from 'node-cron'; /** diff --git a/lib/services/crons/listing-alive-cron.js b/lib/services/crons/listing-alive-cron.js new file mode 100644 index 0000000..60dd5d9 --- /dev/null +++ b/lib/services/crons/listing-alive-cron.js @@ -0,0 +1,13 @@ +import cron from 'node-cron'; +import runActiveChecker from '../listings/listingActiveService.js'; + +async function runTask() { + await runActiveChecker(); +} + +export async function initActiveCheckerCron() { + //run directly on start + await runTask(); + // then every day at 1 am + cron.schedule('0 1 * * *', runTask); +} diff --git a/lib/services/tracking/Tracker-Cron.js b/lib/services/crons/tracker-cron.js similarity index 88% rename from lib/services/tracking/Tracker-Cron.js rename to lib/services/crons/tracker-cron.js index 9b89438..7cfe8cc 100644 --- a/lib/services/tracking/Tracker-Cron.js +++ b/lib/services/crons/tracker-cron.js @@ -1,6 +1,6 @@ import cron from 'node-cron'; import { config, inDevMode } from '../../utils.js'; -import { trackMainEvent } from './Tracker.js'; +import { trackMainEvent } from '../tracking/Tracker.js'; async function runTask() { //make sure to only send tracking events if the user gave us the green light and we are not in dev mode diff --git a/lib/services/immoscout/immoscout-web-translator.js b/lib/services/immoscout/immoscout-web-translator.js index 18a21e0..ec28957 100644 --- a/lib/services/immoscout/immoscout-web-translator.js +++ b/lib/services/immoscout/immoscout-web-translator.js @@ -60,6 +60,7 @@ https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC? */ import queryString from 'query-string'; +import { nullOrEmpty } from '../../utils.js'; const PARAM_NAME_MAP = { heatingtypes: 'heatingtypes', @@ -193,3 +194,14 @@ export function convertWebToMobile(webUrl) { return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`; } + +export function convertImmoscoutListingToMobileListing(url) { + if (nullOrEmpty(url)) { + return null; + } + + return url.replace( + /^https:\/\/www\.immobilienscout24\.de\/expose\//, + 'https://api.mobile.immobilienscout24.de/expose/', + ); +} diff --git a/lib/services/listings/listingActiveService.js b/lib/services/listings/listingActiveService.js new file mode 100644 index 0000000..3dd162d --- /dev/null +++ b/lib/services/listings/listingActiveService.js @@ -0,0 +1,105 @@ +import { deactivateListings, getActiveOrUnknownListings } from '../storage/listingsStorage.js'; +import { getProviders } from '../../utils.js'; +import logger from '../../services/logger.js'; + +/** + * Runs the active-listing checker: + * 1) Loads all listings with unknown or active status. + * 2) Resolves each listing's provider and calls its `activeTester(link)`. + * 3) Collects listings that are no longer active and deactivates them in one batch. + * + * Concurrency: network-bound checks are executed with a configurable concurrency limit. + * + * @param {object} [opts] + * @param {number} [opts.concurrency=8] Max number of parallel activeTester calls. + * @returns {Promise} + */ +export default async function runActiveChecker(opts = {}) { + const { concurrency = 4 } = opts; + + const listings = getActiveOrUnknownListings(); + if (!Array.isArray(listings) || listings.length === 0) { + logger.debug('No listings to check.'); + return; + } + + const providers = await getProviders(); + if (!Array.isArray(providers) || providers.length === 0) { + logger.warn('No providers available. Skipping active checks.'); + return; + } + + // Build a map for O(1) provider lookup by id + /** @type {Record} */ + const providerById = Object.create(null); + for (const p of providers) { + const id = p?.metaInformation?.id; + if (id) providerById[id] = p; + } + + // Small generic mapLimit to cap concurrency without extra deps + /** + * @template T, R + * @param {T[]} items + * @param {number} limit + * @param {(item: T, index: number) => Promise} worker + * @returns {Promise} + */ + async function mapLimit(items, limit, worker) { + const results = new Array(items.length); + let next = 0; + + async function runOne() { + while (next < items.length) { + const i = next++; + try { + results[i] = await worker(items[i], i); + } catch (err) { + results[i] = /** @type {any} */ (err); + } + } + } + + const runners = Array.from({ length: Math.min(limit, items.length) }, runOne); + await Promise.all(runners); + return results; + } + + /** @type {string[]} */ + const listingsSetToInactive = []; + + await mapLimit(listings, concurrency, async (listing) => { + const { provider: listingProviderId, link, id } = listing || {}; + + const matchedProvider = providerById[listingProviderId]; + if (!matchedProvider) { + logger.warn('Could not find matching provider for', listingProviderId); + return; + } + const tester = matchedProvider?.config?.activeTester; + if (typeof tester !== 'function') { + logger.warn('No activeTester configured for', listingProviderId); + return; + } + + // Contract: activeTester(link) returns 1 if active, 0 if inactive + let result; + try { + result = await tester(link); + } catch (err) { + logger.warn('activeTester failed for', listingProviderId, link, err); + return; + } + + if (result === 0 && id) { + listingsSetToInactive.push(id); + } + }); + + if (listingsSetToInactive.length > 0) { + logger.info(`Setting ${listingsSetToInactive.length} listings to inactive.`); + deactivateListings(listingsSetToInactive); + } else { + logger.debug('No listings need to be set inactive.'); + } +} diff --git a/lib/services/listings/listingActiveTester.js b/lib/services/listings/listingActiveTester.js new file mode 100644 index 0000000..148c925 --- /dev/null +++ b/lib/services/listings/listingActiveTester.js @@ -0,0 +1,68 @@ +import fetch from 'node-fetch'; +import { randomBetween, sleep } from '../../utils.js'; + +const maxAttempts = 3; + +/** + * Check if a listing is still active with up to 3 attempts and exponential backoff. + * Backoff waits are capped and the last wait is at most 2000 ms. + * + * Rules: + * - HTTP 200 => return 1 + * - HTTP 401/403 => return -1 (most certainly detected as a bot) + * - HTTP 404 => return 0 + * - Other statuses or network errors => retry until attempts are exhausted + * + * @returns {Promise} 1 if active, o if not active and -1 if detected as bot + */ +export default async function checkIfListingIsActive(link) { + await sleep(randomBetween(50, 100)); + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const res = await fetch(link, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', + 'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8', + }, + }); + + if (res.status === 200) { + return 1; + } + if (res.status === 401) return -1; + if (res.status === 403) return -1; + if (res.status === 404) return 0; + + // For any other status, only retry if attempts remain + if (attempt < maxAttempts) { + await sleep(backoffDelay(attempt)); + continue; + } + + return 0; + } catch { + // Network error: retry if attempts remain + if (attempt < maxAttempts) { + await sleep(backoffDelay(attempt)); + continue; + } + return 0; + } + } + + return 0; +} + +/** + * Exponential backoff delay with cap. + * attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap) + * @param {number} attempt 1-based attempt index + * @returns {number} delay in ms + */ +function backoffDelay(attempt) { + const base = 500; + const cap = 2000; + return Math.min(base * 2 ** (attempt - 1), cap); +} diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index d1b0075..082767b 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -53,6 +53,36 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => { ).map((r) => r.hash); }; +/** + * Return a list of listing that either are active or have an unknown status + * to constantly check if they are still online + * + * @returns {string[]} Array of listings + */ +export const getActiveOrUnknownListings = () => { + return SqliteConnection.query( + `SELECT * + FROM listings + WHERE is_active is null OR is_active = 1 ORDER BY provider`, + ); +}; + +/** + * Deactivates listings by setting is_active = 0 for all matching IDs. + * + * @param {string[]} ids - Array of listing IDs to deactivate. + * @returns {object[]} Result of the SQLite query execution. + */ +export const deactivateListings = (ids) => { + const placeholders = ids.map(() => '?').join(','); + return SqliteConnection.execute( + `UPDATE listings + SET is_active = 0 + WHERE id IN (${placeholders})`, + ids, + ); +}; + /** * Persist a batch of scraped listings for a given job and provider. * @@ -86,9 +116,9 @@ export const storeListings = (jobId, providerId, listings) => { SqliteConnection.withTransaction((db) => { const stmt = db.prepare( `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, - link, created_at) + link, created_at, is_active) VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link, - @created_at) + @created_at, 1) ON CONFLICT(job_id, hash) DO NOTHING`, ); diff --git a/lib/services/storage/migrations/sql/2.active-flag-for-listings.js b/lib/services/storage/migrations/sql/2.active-flag-for-listings.js new file mode 100644 index 0000000..521c467 --- /dev/null +++ b/lib/services/storage/migrations/sql/2.active-flag-for-listings.js @@ -0,0 +1,8 @@ +// Migration: there needs to be a unique index on job_id and hash as only +// this makes the listing indeed unique + +export function up(db) { + db.exec(` + ALTER TABLE listings ADD COLUMN is_active INTEGER DEFAULT 1; + `); +} diff --git a/lib/utils.js b/lib/utils.js index d752339..7f9c3bd 100755 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,5 +1,6 @@ import { dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { readFile } from 'fs/promises'; import { createHash } from 'crypto'; import { DEFAULT_CONFIG } from './defaultConfig.js'; @@ -11,6 +12,26 @@ const RE_GT = />/g; const RE_WEBP = /\/format\/webp/gi; const RE_EXT = /\.(jpe?g|png|gif)(\?.*)?$/i; const HTTPS_PREFIX = 'https://'; +const providersDirectoryPath = `${getDirName()}/provider`; + +/** + * Lazily load all provider modules from the provider directory. + * Caches the resolved array to avoid re-importing on subsequent calls. + * + * @returns {Promise} A list of loaded provider modules. + */ +let cachedProvidersPromise = null; + +export function getProviders() { + if (!cachedProvidersPromise) { + /** @type {string[]} */ + const providerFileNames = fs.readdirSync(providersDirectoryPath).filter((fileName) => fileName.endsWith('.js')); + cachedProvidersPromise = Promise.all( + providerFileNames.map((fileName) => import(pathToFileURL(path.join(providersDirectoryPath, fileName)).href)), + ); + } + return cachedProvidersPromise; +} /** * Safely stringify a value to JSON for storage. @@ -21,7 +42,7 @@ const HTTPS_PREFIX = 'https://'; * @param {T} v - Any JSON-serializable value. * @returns {string|null} JSON string or null. */ -export const toJson = (v) => (v == null ? null : JSON.stringify(v)); +const toJson = (v) => (v == null ? null : JSON.stringify(v)); /** * Safely parse JSON text coming from storage. @@ -33,7 +54,7 @@ export const toJson = (v) => (v == null ? null : JSON.stringify(v)); * @param {T} fallback - Value to return when txt is null/invalid. * @returns {T} Parsed value or fallback. */ -export const fromJson = (txt, fallback) => { +const fromJson = (txt, fallback) => { if (txt == null) return fallback; try { return JSON.parse(txt); @@ -213,23 +234,40 @@ async function getPackageVersion() { return 'N/A'; } +/** + * Sleep helper + * @param {number} ms milliseconds to wait + * @returns {Promise} + */ +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * returns a random into between start and end + * @param a start int + * @param b max int + * @returns {*} + */ +function randomBetween(a, b) { + return Math.floor(Math.random() * (b - a + 1)) + a; +} + +// Call refreshConfig() from the application entrypoint during startup to populate config. await refreshConfig(); -export { isOneOf }; -export { normalizeImageUrl }; -export { inDevMode }; -export { nullOrEmpty }; -export { duringWorkingHoursOrNotSet }; -export { getDirName }; -export { config }; -export { buildHash }; -export { getPackageVersion }; -export default { +export { isOneOf, + normalizeImageUrl, + inDevMode, nullOrEmpty, duringWorkingHoursOrNotSet, getDirName, + sleep, + randomBetween, config, + buildHash, + getPackageVersion, toJson, fromJson, };