External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

upgrading pois

Browse Source
This commit is contained in:
orangecoding
2026-05-12 19:28:58 +02:00
parent bcd3042026
commit 0cbfa25062
10 changed files with 19 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/services/extractor/extractor.js
View File

@@ -29,11 +29,12 @@ export default class Extractor {
* your response will never contain what you are really looking for
* @param url
* @param waitForSelector
* @param jobKey
*/
execute = async (url, waitForSelector = null) => {
execute = async (url, waitForSelector = null, jobKey = null) => {
this.responseText = null;
try {
this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
this.responseText = await puppeteerExtractor(url, waitForSelector, { ...this.options, name: jobKey });
if (this.responseText != null) {
loadParser(this.responseText);
}

6
lib/services/extractor/puppeteerExtractor.js
View File

@@ -148,7 +148,11 @@ export default async function execute(url, waitForSelector, options) {
if (botDetected(pageSource, statusCode)) {
logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
await trackPoi(TRACKING_POIS.DETECTED_AS_BOT);
if (options != null && options.name != null) {
await trackPoi(TRACKING_POIS.DETECTED_AS_BOT + '_' + options.name);
} else {
await trackPoi(TRACKING_POIS.DETECTED_AS_BOT);
}
result = null;
} else {