mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ecbbdd774 | ||
|
|
e1db3840f6 | ||
|
|
26127eeac1 | ||
|
|
90a4ee5dcf | ||
|
|
2aaf63c253 |
2
LICENSE
2
LICENSE
@@ -1,6 +1,6 @@
|
|||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2024 Christian Kellner
|
Copyright (c) 2025 Christian Kellner
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|||||||
@@ -1,118 +1,124 @@
|
|||||||
import {NoNewListingsWarning} from './errors.js';
|
import { NoNewListingsWarning } from './errors.js';
|
||||||
import {setKnownListings, getKnownListings} from './services/storage/listingsStorage.js';
|
import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
|
||||||
import * as notify from './notification/notify.js';
|
import * as notify from './notification/notify.js';
|
||||||
import Extractor from './services/extractor/extractor.js';
|
import Extractor from './services/extractor/extractor.js';
|
||||||
import urlModifier from './services/queryStringMutator.js';
|
import urlModifier from './services/queryStringMutator.js';
|
||||||
|
|
||||||
class FredyRuntime {
|
class FredyRuntime {
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* @param providerConfig the config for the specific provider, we're going to query at the moment
|
* @param providerConfig the config for the specific provider, we're going to query at the moment
|
||||||
* @param notificationConfig the config for all notifications
|
* @param notificationConfig the config for all notifications
|
||||||
* @param providerId the id of the provider currently in use
|
* @param providerId the id of the provider currently in use
|
||||||
* @param jobKey key of the job that is currently running (from within the config)
|
* @param jobKey key of the job that is currently running (from within the config)
|
||||||
* @param similarityCache cache instance holding values to check for similarity of entries
|
* @param similarityCache cache instance holding values to check for similarity of entries
|
||||||
*/
|
*/
|
||||||
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
|
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
|
||||||
this._providerConfig = providerConfig;
|
this._providerConfig = providerConfig;
|
||||||
this._notificationConfig = notificationConfig;
|
this._notificationConfig = notificationConfig;
|
||||||
this._providerId = providerId;
|
this._providerId = providerId;
|
||||||
this._jobKey = jobKey;
|
this._jobKey = jobKey;
|
||||||
this._similarityCache = similarityCache;
|
this._similarityCache = similarityCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
execute() {
|
execute() {
|
||||||
return (
|
return (
|
||||||
//modify the url to make sure search order is correctly set
|
//modify the url to make sure search order is correctly set
|
||||||
Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
|
Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
|
||||||
//scraping the site and try finding new listings
|
//scraping the site and try finding new listings
|
||||||
.then(this._getListings.bind(this))
|
.then(this._getListings.bind(this))
|
||||||
//bring them in a proper form (dictated by the provider)
|
//bring them in a proper form (dictated by the provider)
|
||||||
.then(this._normalize.bind(this))
|
.then(this._normalize.bind(this))
|
||||||
//filter listings with stuff tagged by the blacklist of the provider
|
//filter listings with stuff tagged by the blacklist of the provider
|
||||||
.then(this._filter.bind(this))
|
.then(this._filter.bind(this))
|
||||||
//check if new listings available. if so proceed
|
//check if new listings available. if so proceed
|
||||||
.then(this._findNew.bind(this))
|
.then(this._findNew.bind(this))
|
||||||
//store everything in db
|
//store everything in db
|
||||||
.then(this._save.bind(this))
|
.then(this._save.bind(this))
|
||||||
//check for similar listings. if found, remove them before notifying
|
//check for similar listings. if found, remove them before notifying
|
||||||
.then(this._filterBySimilarListings.bind(this))
|
.then(this._filterBySimilarListings.bind(this))
|
||||||
//notify the user using the configured notification adapter
|
//notify the user using the configured notification adapter
|
||||||
.then(this._notify.bind(this))
|
.then(this._notify.bind(this))
|
||||||
//if an error occurred on the way, handle it here.
|
//if an error occurred on the way, handle it here.
|
||||||
.catch(this._handleError.bind(this))
|
.catch(this._handleError.bind(this))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
_getListings(url) {
|
_getListings(url) {
|
||||||
const extractor = new Extractor();
|
const extractor = new Extractor();
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
extractor.execute(url,this._providerConfig.waitForSelector)
|
extractor
|
||||||
.then(() => {
|
.execute(url, this._providerConfig.waitForSelector)
|
||||||
const listings = extractor.parseResponseText(this._providerConfig.crawlContainer, this._providerConfig.crawlFields);
|
.then(() => {
|
||||||
resolve(listings == null ? [] : listings);
|
const listings = extractor.parseResponseText(
|
||||||
}).catch(err => {
|
this._providerConfig.crawlContainer,
|
||||||
reject(err);
|
this._providerConfig.crawlFields,
|
||||||
/* eslint-disable no-console */
|
url,
|
||||||
console.error(err);
|
);
|
||||||
/* eslint-enable no-console */
|
resolve(listings == null ? [] : listings);
|
||||||
});
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
reject(err);
|
||||||
|
/* eslint-disable no-console */
|
||||||
|
console.error(err);
|
||||||
|
/* eslint-enable no-console */
|
||||||
});
|
});
|
||||||
}
|
});
|
||||||
|
}
|
||||||
|
|
||||||
_normalize(listings) {
|
_normalize(listings) {
|
||||||
return listings.map(this._providerConfig.normalize);
|
return listings.map(this._providerConfig.normalize);
|
||||||
}
|
}
|
||||||
|
|
||||||
_filter(listings) {
|
_filter(listings) {
|
||||||
//only return those where all the fields have been found
|
//only return those where all the fields have been found
|
||||||
const keys = Object.keys(this._providerConfig.crawlFields);
|
const keys = Object.keys(this._providerConfig.crawlFields);
|
||||||
const filteredListings = listings.filter((item) => keys.every((key) => key in item));
|
const filteredListings = listings.filter((item) => keys.every((key) => key in item));
|
||||||
return filteredListings.filter(this._providerConfig.filter);
|
return filteredListings.filter(this._providerConfig.filter);
|
||||||
}
|
}
|
||||||
|
|
||||||
_findNew(listings) {
|
_findNew(listings) {
|
||||||
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
|
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
|
||||||
if (newListings.length === 0) {
|
if (newListings.length === 0) {
|
||||||
throw new NoNewListingsWarning();
|
throw new NoNewListingsWarning();
|
||||||
}
|
|
||||||
return newListings;
|
|
||||||
}
|
}
|
||||||
|
return newListings;
|
||||||
|
}
|
||||||
|
|
||||||
_notify(newListings) {
|
_notify(newListings) {
|
||||||
if (newListings.length === 0) {
|
if (newListings.length === 0) {
|
||||||
throw new NoNewListingsWarning();
|
throw new NoNewListingsWarning();
|
||||||
}
|
|
||||||
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
|
|
||||||
return Promise.all(sendNotifications).then(() => newListings);
|
|
||||||
}
|
}
|
||||||
|
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
|
||||||
|
return Promise.all(sendNotifications).then(() => newListings);
|
||||||
|
}
|
||||||
|
|
||||||
_save(newListings) {
|
_save(newListings) {
|
||||||
const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
|
const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
|
||||||
newListings.forEach((listing) => {
|
newListings.forEach((listing) => {
|
||||||
currentListings[listing.id] = Date.now();
|
currentListings[listing.id] = Date.now();
|
||||||
});
|
});
|
||||||
setKnownListings(this._jobKey, this._providerId, currentListings);
|
setKnownListings(this._jobKey, this._providerId, currentListings);
|
||||||
return newListings;
|
return newListings;
|
||||||
}
|
}
|
||||||
|
|
||||||
_filterBySimilarListings(listings) {
|
_filterBySimilarListings(listings) {
|
||||||
const filteredList = listings.filter((listing) => {
|
const filteredList = listings.filter((listing) => {
|
||||||
const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
|
const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
|
||||||
if (similar) {
|
if (similar) {
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
|
console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
|
||||||
/* eslint-enable no-console */
|
/* eslint-enable no-console */
|
||||||
}
|
}
|
||||||
return !similar;
|
return !similar;
|
||||||
});
|
});
|
||||||
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
|
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
|
||||||
return filteredList;
|
return filteredList;
|
||||||
}
|
}
|
||||||
|
|
||||||
_handleError(err) {
|
_handleError(err) {
|
||||||
if (err.name !== 'NoNewListingsWarning') console.error(err);
|
if (err.name !== 'NoNewListingsWarning') console.error(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default FredyRuntime;
|
export default FredyRuntime;
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
import restana from 'restana';
|
import restana from 'restana';
|
||||||
import fetch from 'node-fetch';
|
|
||||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||||
import * as userStorage from '../../services/storage/userStorage.js';
|
import * as userStorage from '../../services/storage/userStorage.js';
|
||||||
import * as immoscoutProvider from '../../provider/immoscout.js';
|
|
||||||
import { config } from '../../utils.js';
|
import { config } from '../../utils.js';
|
||||||
import { isAdmin } from '../security.js';
|
import { isAdmin } from '../security.js';
|
||||||
import {trackDemoJobCreated} from '../../services/tracking/Tracker.js';
|
import { trackDemoJobCreated } from '../../services/tracking/Tracker.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const jobRouter = service.newRouter();
|
const jobRouter = service.newRouter();
|
||||||
function doesJobBelongsToUser(job, req) {
|
function doesJobBelongsToUser(job, req) {
|
||||||
@@ -28,7 +26,7 @@ jobRouter.get('/', async (req, res) => {
|
|||||||
jobRouter.get('/processingTimes', async (req, res) => {
|
jobRouter.get('/processingTimes', async (req, res) => {
|
||||||
res.body = {
|
res.body = {
|
||||||
interval: config.interval,
|
interval: config.interval,
|
||||||
lastRun: config.lastRun || null
|
lastRun: config.lastRun || null,
|
||||||
};
|
};
|
||||||
res.send();
|
res.send();
|
||||||
});
|
});
|
||||||
@@ -51,7 +49,7 @@ jobRouter.post('/', async (req, res) => {
|
|||||||
trackDemoJobCreated({
|
trackDemoJobCreated({
|
||||||
name,
|
name,
|
||||||
provider,
|
provider,
|
||||||
adapter: notificationAdapter
|
adapter: notificationAdapter,
|
||||||
});
|
});
|
||||||
res.send();
|
res.send();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import utils, {buildHash} from '../utils.js';
|
import utils, { buildHash } from '../utils.js';
|
||||||
let appliedBlackList = [];
|
let appliedBlackList = [];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -26,7 +26,7 @@ const config = {
|
|||||||
url: null,
|
url: null,
|
||||||
crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
|
crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
|
||||||
sortByDateParam: 'sortby=19',
|
sortByDateParam: 'sortby=19',
|
||||||
waitForSelector: 'div[data-testid="serp-core-classified-card-testid"]',
|
waitForSelector: 'div[data-testid="serp-resultscount-testid"]',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
|
id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
|
||||||
title: 'button@title |trim',
|
title: 'button@title |trim',
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ const config = {
|
|||||||
crawlContainer:
|
crawlContainer:
|
||||||
'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
|
'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
|
||||||
sortByDateParam: 'order=DateDesc',
|
sortByDateParam: 'order=DateDesc',
|
||||||
waitForSelector: 'div[data-testid="cardmfe-price-testid"]',
|
waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: 'a@href',
|
id: 'a@href',
|
||||||
price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
|
price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
|
||||||
|
|||||||
@@ -1,45 +1,43 @@
|
|||||||
import {setDebug} from './utils.js';
|
import { setDebug } from './utils.js';
|
||||||
import puppeteerExtractor from './puppeteerExtractor.js';
|
import puppeteerExtractor from './puppeteerExtractor.js';
|
||||||
import {loadParser, parse} from './parser/parser.js';
|
import { loadParser, parse } from './parser/parser.js';
|
||||||
|
|
||||||
const DEFAULT_OPTIONS = {
|
const DEFAULT_OPTIONS = {
|
||||||
debug: false,
|
debug: false,
|
||||||
puppeteerTimeout: 20_000,
|
puppeteerTimeout: 60_000,
|
||||||
puppeteerHeadless: true
|
puppeteerHeadless: true,
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export default class Extractor {
|
export default class Extractor {
|
||||||
constructor(options) {
|
constructor(options) {
|
||||||
this.options = {
|
this.options = {
|
||||||
...DEFAULT_OPTIONS,
|
...DEFAULT_OPTIONS,
|
||||||
...options
|
...options,
|
||||||
};
|
};
|
||||||
this.responseText = null;
|
this.responseText = null;
|
||||||
setDebug(this.options);
|
setDebug(this.options);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* if you are extracting data from a SPA, you must provide a selector, otherwise
|
||||||
|
* your response will never contain what you are really looking for
|
||||||
|
* @param url
|
||||||
|
* @param waitForSelector
|
||||||
|
*/
|
||||||
|
execute = async (url, waitForSelector = null) => {
|
||||||
|
this.responseText = null;
|
||||||
|
try {
|
||||||
|
this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
|
||||||
|
if (this.responseText != null) {
|
||||||
|
loadParser(this.responseText);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error trying to load page.', error);
|
||||||
}
|
}
|
||||||
|
return this;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
parseResponseText = (crawlContainer, crawlFields, url) => {
|
||||||
* if you are extracting data from a SPA, you must provide a selector, otherwise
|
return parse(crawlContainer, crawlFields, this.responseText, url);
|
||||||
* your response will never contain what you are really looking for
|
};
|
||||||
* @param url
|
|
||||||
* @param waitForSelector
|
|
||||||
*/
|
|
||||||
execute = async (url, waitForSelector = null) => {
|
|
||||||
this.responseText = null;
|
|
||||||
try {
|
|
||||||
this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
|
|
||||||
if(this.responseText != null) {
|
|
||||||
loadParser(this.responseText);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error trying to load page.', error);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
parseResponseText = (crawlContainer, crawlFields) => {
|
|
||||||
return parse(crawlContainer, crawlFields, this.responseText);
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,92 +3,95 @@ import * as cheerio from 'cheerio';
|
|||||||
let $ = null;
|
let $ = null;
|
||||||
|
|
||||||
export function loadParser(text) {
|
export function loadParser(text) {
|
||||||
$ = cheerio.load(text);
|
$ = cheerio.load(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function parse(crawlContainer, crawlFields, text) {
|
export function parse(crawlContainer, crawlFields, text, url) {
|
||||||
if (!text) {
|
if (!text) {
|
||||||
console.warn('Cannot parse, text was empty.');
|
console.warn('Cannot parse, text was empty for url ', url);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!crawlContainer || !crawlFields) {
|
if (!crawlContainer || !crawlFields) {
|
||||||
console.warn('Cannot parse, selector was empty.');
|
console.warn('Cannot parse, selector was empty for url ', url);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = [];
|
const result = [];
|
||||||
|
|
||||||
if ($(crawlContainer).length === 0) {
|
if ($(crawlContainer).length === 0) {
|
||||||
console.error('No elements in crawl container found!');
|
console.warn('No elements in crawl container found for url ', url);
|
||||||
}
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
$(crawlContainer).each((_, element) => {
|
$(crawlContainer).each((_, element) => {
|
||||||
const container = $(element);
|
const container = $(element);
|
||||||
const parsedObject = {};
|
const parsedObject = {};
|
||||||
|
|
||||||
// Parse fields based on crawlFields
|
// Parse fields based on crawlFields
|
||||||
for (const [key, fieldSelector] of Object.entries(crawlFields)) {
|
for (const [key, fieldSelector] of Object.entries(crawlFields)) {
|
||||||
let value;
|
let value;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const selector = fieldSelector.includes('|')
|
||||||
|
? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim()
|
||||||
|
: fieldSelector;
|
||||||
|
|
||||||
const selector = fieldSelector.includes('|') ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim() : fieldSelector;
|
if (selector.includes('@')) {
|
||||||
|
const [sel, attr] = selector.split('@');
|
||||||
if (selector.includes('@')) {
|
if (sel.length === 0) {
|
||||||
const [sel, attr] = selector.split('@');
|
value = container.attr(attr.trim());
|
||||||
if (sel.length === 0) {
|
} else {
|
||||||
value = container.attr(attr.trim());
|
value = container.find(sel.trim()).attr(attr.trim());
|
||||||
} else {
|
}
|
||||||
value = container.find(sel.trim()).attr(attr.trim());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
value = container.find(selector.trim()).text();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply modifiers if specified
|
|
||||||
if (fieldSelector.includes('|')) {
|
|
||||||
const [_, ...modifiers] = fieldSelector.split('|').map(s => s.trim());
|
|
||||||
value = applyModifiers(value, modifiers);
|
|
||||||
}
|
|
||||||
|
|
||||||
parsedObject[key] = value || null;
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
|
|
||||||
parsedObject[key] = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parsedObject.id != null) {
|
|
||||||
result.push(parsedObject);
|
|
||||||
} else {
|
} else {
|
||||||
console.warn('ID not found. Not relaying object.');
|
value = container.find(selector.trim()).text();
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
|
||||||
return result;
|
// Apply modifiers if specified
|
||||||
|
if (fieldSelector.includes('|')) {
|
||||||
|
/* eslint-disable no-unused-vars */
|
||||||
|
const [_, ...modifiers] = fieldSelector.split('|').map((s) => s.trim());
|
||||||
|
/* eslint-disable no-unused-vars */
|
||||||
|
value = applyModifiers(value, modifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
parsedObject[key] = value || null;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
|
||||||
|
parsedObject[key] = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsedObject.id != null) {
|
||||||
|
result.push(parsedObject);
|
||||||
|
} else {
|
||||||
|
console.warn('ID not found. Not relaying object.');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper function to apply modifiers
|
// Helper function to apply modifiers
|
||||||
function applyModifiers(value, modifiers) {
|
function applyModifiers(value, modifiers) {
|
||||||
if (!value) return value;
|
if (!value) return value;
|
||||||
|
|
||||||
modifiers.forEach(modifier => {
|
modifiers.forEach((modifier) => {
|
||||||
switch (modifier) {
|
switch (modifier) {
|
||||||
case 'int':
|
case 'int':
|
||||||
value = parseInt(value, 10);
|
value = parseInt(value, 10);
|
||||||
break;
|
break;
|
||||||
case 'trim':
|
case 'trim':
|
||||||
value = value.replace(/\s+/g, ' ').trim();
|
value = value.replace(/\s+/g, ' ').trim();
|
||||||
break;
|
break;
|
||||||
case 'removeNewline':
|
case 'removeNewline':
|
||||||
value = value.replace(/\n/g, ' ');
|
value = value.replace(/\n/g, ' ');
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
console.warn(`Unknown modifier: ${modifier}`);
|
console.warn(`Unknown modifier: ${modifier}`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,48 +1,49 @@
|
|||||||
import puppeteer from 'puppeteer-extra';
|
import puppeteer from 'puppeteer-extra';
|
||||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||||
import {debug, DEFAULT_HEADER, botDetected} from './utils.js';
|
import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
|
||||||
|
|
||||||
puppeteer.use(StealthPlugin());
|
puppeteer.use(StealthPlugin());
|
||||||
|
|
||||||
export default async function execute(url, waitForSelector, options) {
|
export default async function execute(url, waitForSelector, options) {
|
||||||
let browser;
|
let browser;
|
||||||
try {
|
try {
|
||||||
debug(`Sending request to ${url} using Puppeteer.`);
|
debug(`Sending request to ${url} using Puppeteer.`);
|
||||||
|
|
||||||
browser = await puppeteer.launch({
|
browser = await puppeteer.launch({
|
||||||
headless: options.puppeteerHeadless ?? true,
|
headless: options.puppeteerHeadless ?? true,
|
||||||
args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox']
|
args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox'],
|
||||||
});
|
timeout: options.puppeteerTimeout || 30_000,
|
||||||
let page = await browser.newPage();
|
});
|
||||||
await page.setExtraHTTPHeaders(DEFAULT_HEADER);
|
let page = await browser.newPage();
|
||||||
const response = await page.goto(url, {
|
await page.setExtraHTTPHeaders(DEFAULT_HEADER);
|
||||||
waitUntil: 'domcontentloaded'
|
const response = await page.goto(url, {
|
||||||
});
|
waitUntil: 'domcontentloaded',
|
||||||
let pageSource;
|
});
|
||||||
//if we're extracting data from a spa, we must wait for the selector
|
let pageSource;
|
||||||
if (waitForSelector != null) {
|
//if we're extracting data from a spa, we must wait for the selector
|
||||||
await page.waitForSelector(waitForSelector);
|
if (waitForSelector != null) {
|
||||||
pageSource = await page.evaluate(selector => {
|
await page.waitForSelector(waitForSelector);
|
||||||
return document.querySelector(selector).innerHTML;
|
pageSource = await page.evaluate((selector) => {
|
||||||
}, waitForSelector);
|
return document.querySelector(selector).innerHTML;
|
||||||
} else {
|
}, waitForSelector);
|
||||||
pageSource = await page.content();
|
} else {
|
||||||
}
|
pageSource = await page.content();
|
||||||
|
|
||||||
const statusCode = response.status();
|
|
||||||
|
|
||||||
if (botDetected(pageSource, statusCode)) {
|
|
||||||
console.warn('We have been detected as a bot :-/ Tried url: => ', url);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return await page.content();
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error executing with puppeteer executor', error);
|
|
||||||
return null;
|
|
||||||
} finally {
|
|
||||||
if (browser != null) {
|
|
||||||
await browser.close();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const statusCode = response.status();
|
||||||
|
|
||||||
|
if (botDetected(pageSource, statusCode)) {
|
||||||
|
console.warn('We have been detected as a bot :-/ Tried url: => ', url);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await page.content();
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error executing with puppeteer executor', error);
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (browser != null) {
|
||||||
|
await browser.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,35 +1,32 @@
|
|||||||
let debuggingOn = false;
|
let debuggingOn = false;
|
||||||
|
|
||||||
export const DEFAULT_HEADER = {
|
export const DEFAULT_HEADER = {
|
||||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||||
'Accept-Language': 'en-US,en;q=0.5',
|
'Accept-Language': 'en-US,en;q=0.5',
|
||||||
'Connection': 'keep-alive',
|
Connection: 'keep-alive',
|
||||||
'Upgrade-Insecure-Requests': '1',
|
'Upgrade-Insecure-Requests': '1',
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
|
'User-Agent':
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
|
||||||
};
|
};
|
||||||
|
|
||||||
export const setDebug = options => {
|
export const setDebug = (options) => {
|
||||||
debuggingOn = !!options?.debug;
|
debuggingOn = !!options?.debug;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const debug = (message) => {
|
export const debug = (message) => {
|
||||||
if(debuggingOn) {
|
if (debuggingOn) {
|
||||||
console.debug(message);
|
/* eslint-disable no-console */
|
||||||
}
|
console.debug(message);
|
||||||
|
/* eslint-enable no-console */
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const botDetected = (pageSource, statusCode) => {
|
export const botDetected = (pageSource, statusCode) => {
|
||||||
const suspiciousStatusCodes = [
|
const suspiciousStatusCodes = [403, 429];
|
||||||
403, 429
|
const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
|
||||||
];
|
|
||||||
const botDetectionPatterns = [
|
|
||||||
/verify you are human/i,
|
|
||||||
/access denied/i,
|
|
||||||
/x-amz-cf-id/i,
|
|
||||||
];
|
|
||||||
|
|
||||||
const detectedInSource = botDetectionPatterns.some(pattern => pattern.test(pageSource));
|
const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
|
||||||
const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
|
const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
|
||||||
|
|
||||||
return detectedInSource || detectedByStatus;
|
return detectedInSource || detectedByStatus;
|
||||||
};
|
};
|
||||||
12
package.json
12
package.json
@@ -50,17 +50,17 @@
|
|||||||
"Firefox ESR"
|
"Firefox ESR"
|
||||||
],
|
],
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@douyinfe/semi-ui": "2.71.3",
|
"@douyinfe/semi-ui": "2.72.3",
|
||||||
"@rematch/core": "2.2.0",
|
"@rematch/core": "2.2.0",
|
||||||
"@rematch/loading": "2.1.2",
|
"@rematch/loading": "2.1.2",
|
||||||
"@sendgrid/mail": "8.1.4",
|
"@sendgrid/mail": "8.1.4",
|
||||||
"@vitejs/plugin-react": "4.3.4",
|
"@vitejs/plugin-react": "4.3.4",
|
||||||
"better-sqlite3": "^11.7.0",
|
"better-sqlite3": "^11.7.2",
|
||||||
"body-parser": "1.20.3",
|
"body-parser": "1.20.3",
|
||||||
"cheerio": "^1.0.0",
|
"cheerio": "^1.0.0",
|
||||||
"cookie-session": "2.1.0",
|
"cookie-session": "2.1.0",
|
||||||
"handlebars": "4.7.8",
|
"handlebars": "4.7.8",
|
||||||
"highcharts": "12.1.0",
|
"highcharts": "12.1.2",
|
||||||
"highcharts-react-official": "3.2.1",
|
"highcharts-react-official": "3.2.1",
|
||||||
"lodash": "4.17.21",
|
"lodash": "4.17.21",
|
||||||
"lowdb": "6.0.1",
|
"lowdb": "6.0.1",
|
||||||
@@ -70,7 +70,7 @@
|
|||||||
"node-fetch": "3.3.2",
|
"node-fetch": "3.3.2",
|
||||||
"node-mailjet": "6.0.6",
|
"node-mailjet": "6.0.6",
|
||||||
"package-up": "^5.0.0",
|
"package-up": "^5.0.0",
|
||||||
"puppeteer": "^23.10.4",
|
"puppeteer": "^23.11.1",
|
||||||
"puppeteer-extra": "^3.3.6",
|
"puppeteer-extra": "^3.3.6",
|
||||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||||
"query-string": "9.1.1",
|
"query-string": "9.1.1",
|
||||||
@@ -95,12 +95,12 @@
|
|||||||
"chai": "5.1.2",
|
"chai": "5.1.2",
|
||||||
"eslint": "8.56.0",
|
"eslint": "8.56.0",
|
||||||
"eslint-config-prettier": "8.8.0",
|
"eslint-config-prettier": "8.8.0",
|
||||||
"eslint-plugin-react": "7.37.2",
|
"eslint-plugin-react": "7.37.3",
|
||||||
"esmock": "2.6.9",
|
"esmock": "2.6.9",
|
||||||
"history": "5.3.0",
|
"history": "5.3.0",
|
||||||
"husky": "9.1.7",
|
"husky": "9.1.7",
|
||||||
"less": "4.2.1",
|
"less": "4.2.1",
|
||||||
"lint-staged": "15.2.11",
|
"lint-staged": "15.3.0",
|
||||||
"mocha": "10.8.2",
|
"mocha": "10.8.2",
|
||||||
"prettier": "3.4.2",
|
"prettier": "3.4.2",
|
||||||
"redux-logger": "3.0.6"
|
"redux-logger": "3.0.6"
|
||||||
|
|||||||
Reference in New Issue
Block a user