External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

better logging, fixing code smells

Browse Source
This commit is contained in:
Christian Kellner
2025-01-07 12:25:19 +01:00
parent 2aaf63c253
commit 90a4ee5dcf
5 changed files with 232 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
lib/FredyRuntime.js
View File

@@ -1,118 +1,124 @@
import {NoNewListingsWarning} from './errors.js'; import { NoNewListingsWarning } from './errors.js';
import {setKnownListings, getKnownListings} from './services/storage/listingsStorage.js'; import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
import * as notify from './notification/notify.js'; import * as notify from './notification/notify.js';
import Extractor from './services/extractor/extractor.js'; import Extractor from './services/extractor/extractor.js';
import urlModifier from './services/queryStringMutator.js'; import urlModifier from './services/queryStringMutator.js';
class FredyRuntime { class FredyRuntime {
/** /**
* *
* @param providerConfig the config for the specific provider, we're going to query at the moment * @param providerConfig the config for the specific provider, we're going to query at the moment
* @param notificationConfig the config for all notifications * @param notificationConfig the config for all notifications
* @param providerId the id of the provider currently in use * @param providerId the id of the provider currently in use
* @param jobKey key of the job that is currently running (from within the config) * @param jobKey key of the job that is currently running (from within the config)
* @param similarityCache cache instance holding values to check for similarity of entries * @param similarityCache cache instance holding values to check for similarity of entries
*/ */
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) { constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
this._providerConfig = providerConfig; this._providerConfig = providerConfig;
this._notificationConfig = notificationConfig; this._notificationConfig = notificationConfig;
this._providerId = providerId; this._providerId = providerId;
this._jobKey = jobKey; this._jobKey = jobKey;
this._similarityCache = similarityCache; this._similarityCache = similarityCache;
} }
execute() { execute() {
return ( return (
//modify the url to make sure search order is correctly set //modify the url to make sure search order is correctly set
Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam)) Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
//scraping the site and try finding new listings //scraping the site and try finding new listings
.then(this._getListings.bind(this)) .then(this._getListings.bind(this))
//bring them in a proper form (dictated by the provider) //bring them in a proper form (dictated by the provider)
.then(this._normalize.bind(this)) .then(this._normalize.bind(this))
//filter listings with stuff tagged by the blacklist of the provider //filter listings with stuff tagged by the blacklist of the provider
.then(this._filter.bind(this)) .then(this._filter.bind(this))
//check if new listings available. if so proceed //check if new listings available. if so proceed
.then(this._findNew.bind(this)) .then(this._findNew.bind(this))
//store everything in db //store everything in db
.then(this._save.bind(this)) .then(this._save.bind(this))
//check for similar listings. if found, remove them before notifying //check for similar listings. if found, remove them before notifying
.then(this._filterBySimilarListings.bind(this)) .then(this._filterBySimilarListings.bind(this))
//notify the user using the configured notification adapter //notify the user using the configured notification adapter
.then(this._notify.bind(this)) .then(this._notify.bind(this))
//if an error occurred on the way, handle it here. //if an error occurred on the way, handle it here.
.catch(this._handleError.bind(this)) .catch(this._handleError.bind(this))
); );
} }
_getListings(url) { _getListings(url) {
const extractor = new Extractor(); const extractor = new Extractor();
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
extractor.execute(url,this._providerConfig.waitForSelector) extractor
.then(() => { .execute(url, this._providerConfig.waitForSelector)
const listings = extractor.parseResponseText(this._providerConfig.crawlContainer, this._providerConfig.crawlFields); .then(() => {
resolve(listings == null ? [] : listings); const listings = extractor.parseResponseText(
}).catch(err => { this._providerConfig.crawlContainer,
reject(err); this._providerConfig.crawlFields,
/* eslint-disable no-console */ url,
console.error(err); );
/* eslint-enable no-console */ resolve(listings == null ? [] : listings);
}); })
.catch((err) => {
reject(err);
/* eslint-disable no-console */
console.error(err);
/* eslint-enable no-console */
}); });
} });
}
_normalize(listings) { _normalize(listings) {
return listings.map(this._providerConfig.normalize); return listings.map(this._providerConfig.normalize);
} }
_filter(listings) { _filter(listings) {
//only return those where all the fields have been found //only return those where all the fields have been found
const keys = Object.keys(this._providerConfig.crawlFields); const keys = Object.keys(this._providerConfig.crawlFields);
const filteredListings = listings.filter((item) => keys.every((key) => key in item)); const filteredListings = listings.filter((item) => keys.every((key) => key in item));
return filteredListings.filter(this._providerConfig.filter); return filteredListings.filter(this._providerConfig.filter);
} }
_findNew(listings) { _findNew(listings) {
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null); const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
if (newListings.length === 0) { if (newListings.length === 0) {
throw new NoNewListingsWarning(); throw new NoNewListingsWarning();
}
return newListings;
} }
return newListings;
}
_notify(newListings) { _notify(newListings) {
if (newListings.length === 0) { if (newListings.length === 0) {
throw new NoNewListingsWarning(); throw new NoNewListingsWarning();
}
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
return Promise.all(sendNotifications).then(() => newListings);
} }
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
return Promise.all(sendNotifications).then(() => newListings);
}
_save(newListings) { _save(newListings) {
const currentListings = getKnownListings(this._jobKey, this._providerId) || {}; const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
newListings.forEach((listing) => { newListings.forEach((listing) => {
currentListings[listing.id] = Date.now(); currentListings[listing.id] = Date.now();
}); });
setKnownListings(this._jobKey, this._providerId, currentListings); setKnownListings(this._jobKey, this._providerId, currentListings);
return newListings; return newListings;
} }
_filterBySimilarListings(listings) { _filterBySimilarListings(listings) {
const filteredList = listings.filter((listing) => { const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title); const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
if (similar) { if (similar) {
/* eslint-disable no-console */ /* eslint-disable no-console */
console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title); console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
/* eslint-enable no-console */ /* eslint-enable no-console */
} }
return !similar; return !similar;
}); });
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title)); filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
return filteredList; return filteredList;
} }
_handleError(err) { _handleError(err) {
if (err.name !== 'NoNewListingsWarning') console.error(err); if (err.name !== 'NoNewListingsWarning') console.error(err);
} }
} }
export default FredyRuntime; export default FredyRuntime;

8
lib/api/routes/jobRouter.js
View File

@@ -1,11 +1,9 @@
import restana from 'restana'; import restana from 'restana';
import fetch from 'node-fetch';
import * as jobStorage from '../../services/storage/jobStorage.js'; import * as jobStorage from '../../services/storage/jobStorage.js';
import * as userStorage from '../../services/storage/userStorage.js'; import * as userStorage from '../../services/storage/userStorage.js';
import * as immoscoutProvider from '../../provider/immoscout.js';
import { config } from '../../utils.js'; import { config } from '../../utils.js';
import { isAdmin } from '../security.js'; import { isAdmin } from '../security.js';
import {trackDemoJobCreated} from '../../services/tracking/Tracker.js'; import { trackDemoJobCreated } from '../../services/tracking/Tracker.js';
const service = restana(); const service = restana();
const jobRouter = service.newRouter(); const jobRouter = service.newRouter();
function doesJobBelongsToUser(job, req) { function doesJobBelongsToUser(job, req) {
@@ -28,7 +26,7 @@ jobRouter.get('/', async (req, res) => {
jobRouter.get('/processingTimes', async (req, res) => { jobRouter.get('/processingTimes', async (req, res) => {
res.body = { res.body = {
interval: config.interval, interval: config.interval,
lastRun: config.lastRun || null lastRun: config.lastRun || null,
}; };
res.send(); res.send();
}); });
@@ -51,7 +49,7 @@ jobRouter.post('/', async (req, res) => {
trackDemoJobCreated({ trackDemoJobCreated({
name, name,
provider, provider,
adapter: notificationAdapter adapter: notificationAdapter,
}); });
res.send(); res.send();
}); });

70
lib/services/extractor/extractor.js
View File

@@ -1,45 +1,43 @@
import {setDebug} from './utils.js'; import { setDebug } from './utils.js';
import puppeteerExtractor from './puppeteerExtractor.js'; import puppeteerExtractor from './puppeteerExtractor.js';
import {loadParser, parse} from './parser/parser.js'; import { loadParser, parse } from './parser/parser.js';
const DEFAULT_OPTIONS = { const DEFAULT_OPTIONS = {
debug: false, debug: false,
puppeteerTimeout: 20_000, puppeteerTimeout: 60_000,
puppeteerHeadless: true puppeteerHeadless: true,
}; };
export default class Extractor { export default class Extractor {
constructor(options) { constructor(options) {
this.options = { this.options = {
...DEFAULT_OPTIONS, ...DEFAULT_OPTIONS,
...options ...options,
}; };
this.responseText = null; this.responseText = null;
setDebug(this.options); setDebug(this.options);
}
/**
* if you are extracting data from a SPA, you must provide a selector, otherwise
* your response will never contain what you are really looking for
* @param url
* @param waitForSelector
*/
execute = async (url, waitForSelector = null) => {
this.responseText = null;
try {
this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
if (this.responseText != null) {
loadParser(this.responseText);
}
} catch (error) {
console.error('Error trying to load page.', error);
} }
return this;
};
/** parseResponseText = (crawlContainer, crawlFields, url) => {
* if you are extracting data from a SPA, you must provide a selector, otherwise return parse(crawlContainer, crawlFields, this.responseText, url);
* your response will never contain what you are really looking for };
* @param url
* @param waitForSelector
*/
execute = async (url, waitForSelector = null) => {
this.responseText = null;
try {
this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
if(this.responseText != null) {
loadParser(this.responseText);
}
} catch (error) {
console.error('Error trying to load page.', error);
}
return this;
};
parseResponseText = (crawlContainer, crawlFields) => {
return parse(crawlContainer, crawlFields, this.responseText);
};
} }

142
lib/services/extractor/parser/parser.js
View File

@@ -3,92 +3,94 @@ import * as cheerio from 'cheerio';
let $ = null; let $ = null;
export function loadParser(text) { export function loadParser(text) {
$ = cheerio.load(text); $ = cheerio.load(text);
} }
export function parse(crawlContainer, crawlFields, text) { export function parse(crawlContainer, crawlFields, text, url) {
if (!text) { if (!text) {
console.warn('Cannot parse, text was empty.'); console.warn('Cannot parse, text was empty for url ', url);
return null; return null;
} }
if (!crawlContainer || !crawlFields) { if (!crawlContainer || !crawlFields) {
console.warn('Cannot parse, selector was empty.'); console.warn('Cannot parse, selector was empty for url ', url);
return null; return null;
} }
const result = []; const result = [];
if ($(crawlContainer).length === 0) { if ($(crawlContainer).length === 0) {
console.error('No elements in crawl container found!'); console.error('No elements in crawl container found for url ', url);
} }
$(crawlContainer).each((_, element) => { $(crawlContainer).each((_, element) => {
const container = $(element); const container = $(element);
const parsedObject = {}; const parsedObject = {};
// Parse fields based on crawlFields // Parse fields based on crawlFields
for (const [key, fieldSelector] of Object.entries(crawlFields)) { for (const [key, fieldSelector] of Object.entries(crawlFields)) {
let value; let value;
try { try {
const selector = fieldSelector.includes('|')
? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim()
: fieldSelector;
const selector = fieldSelector.includes('|') ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim() : fieldSelector; if (selector.includes('@')) {
const [sel, attr] = selector.split('@');
if (selector.includes('@')) { if (sel.length === 0) {
const [sel, attr] = selector.split('@'); value = container.attr(attr.trim());
if (sel.length === 0) { } else {
value = container.attr(attr.trim()); value = container.find(sel.trim()).attr(attr.trim());
} else { }
value = container.find(sel.trim()).attr(attr.trim());
}
} else {
value = container.find(selector.trim()).text();
}
// Apply modifiers if specified
if (fieldSelector.includes('|')) {
const [_, ...modifiers] = fieldSelector.split('|').map(s => s.trim());
value = applyModifiers(value, modifiers);
}
parsedObject[key] = value || null;
} catch (error) {
console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
parsedObject[key] = null;
}
}
if (parsedObject.id != null) {
result.push(parsedObject);
} else { } else {
console.warn('ID not found. Not relaying object.'); value = container.find(selector.trim()).text();
} }
});
return result; // Apply modifiers if specified
if (fieldSelector.includes('|')) {
/* eslint-disable no-unused-vars */
const [_, ...modifiers] = fieldSelector.split('|').map((s) => s.trim());
/* eslint-disable no-unused-vars */
value = applyModifiers(value, modifiers);
}
parsedObject[key] = value || null;
} catch (error) {
console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
parsedObject[key] = null;
}
}
if (parsedObject.id != null) {
result.push(parsedObject);
} else {
console.warn('ID not found. Not relaying object.');
}
});
return result;
} }
// Helper function to apply modifiers // Helper function to apply modifiers
function applyModifiers(value, modifiers) { function applyModifiers(value, modifiers) {
if (!value) return value; if (!value) return value;
modifiers.forEach(modifier => { modifiers.forEach((modifier) => {
switch (modifier) { switch (modifier) {
case 'int': case 'int':
value = parseInt(value, 10); value = parseInt(value, 10);
break; break;
case 'trim': case 'trim':
value = value.replace(/\s+/g, ' ').trim(); value = value.replace(/\s+/g, ' ').trim();
break; break;
case 'removeNewline': case 'removeNewline':
value = value.replace(/\n/g, ' '); value = value.replace(/\n/g, ' ');
break; break;
default: default:
console.warn(`Unknown modifier: ${modifier}`); console.warn(`Unknown modifier: ${modifier}`);
} }
}); });
return value; return value;
} }

39
lib/services/extractor/utils.js
View File

@@ -1,35 +1,32 @@
let debuggingOn = false; let debuggingOn = false;
export const DEFAULT_HEADER = { export const DEFAULT_HEADER = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5', 'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive', Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1', 'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' 'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
}; };
export const setDebug = options => { export const setDebug = (options) => {
debuggingOn = !!options?.debug; debuggingOn = !!options?.debug;
}; };
export const debug = (message) => { export const debug = (message) => {
if(debuggingOn) { if (debuggingOn) {
console.debug(message); /* eslint-disable no-console */
} console.debug(message);
/* eslint-enable no-console */
}
}; };
export const botDetected = (pageSource, statusCode) => { export const botDetected = (pageSource, statusCode) => {
const suspiciousStatusCodes = [ const suspiciousStatusCodes = [403, 429];
403, 429 const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
];
const botDetectionPatterns = [
/verify you are human/i,
/access denied/i,
/x-amz-cf-id/i,
];
const detectedInSource = botDetectionPatterns.some(pattern => pattern.test(pageSource)); const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
const detectedByStatus = suspiciousStatusCodes.includes(statusCode); const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
return detectedInSource || detectedByStatus; return detectedInSource || detectedByStatus;
}; };