External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

better logging, fixing code smells

Browse Source
This commit is contained in:
Christian Kellner
2025-01-07 12:25:19 +01:00
parent 2aaf63c253
commit 90a4ee5dcf
5 changed files with 232 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
lib/services/extractor/utils.js
View File

@@ -1,35 +1,32 @@
let debuggingOn = false;
export const DEFAULT_HEADER = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
};
export const setDebug = options => {
debuggingOn = !!options?.debug;
export const setDebug = (options) => {
debuggingOn = !!options?.debug;
};
export const debug = (message) => {
if(debuggingOn) {
console.debug(message);
}
if (debuggingOn) {
/* eslint-disable no-console */
console.debug(message);
/* eslint-enable no-console */
}
};
export const botDetected = (pageSource, statusCode) => {
const suspiciousStatusCodes = [
403, 429
];
const botDetectionPatterns = [
/verify you are human/i,
/access denied/i,
/x-amz-cf-id/i,
];
const suspiciousStatusCodes = [403, 429];
const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
const detectedInSource = botDetectionPatterns.some(pattern => pattern.test(pageSource));
const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
return detectedInSource || detectedByStatus;
};
return detectedInSource || detectedByStatus;
};