External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Making Fredy an ESM project (#70)

Browse Source 
Making Fredy an ESM project
This commit is contained in:
Christian Kellner
2023-03-13 13:42:43 +01:00
committed by GitHub
parent 7d0ec72a0c
commit 2c5eceb0c1
69 changed files with 862 additions and 1567 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
lib/services/scraper.js
View File

@@ -1,7 +1,6 @@
const config = require('../../conf/config.json');
const makeDriver = require('./requestDriver');
const Xray = require('x-ray');
import { config } from '../utils.js';
import makeDriver from './requestDriver.js';
import Xray from 'x-ray';
class Scraper {
constructor() {
const filters = {
@@ -9,38 +8,29 @@ class Scraper {
trim: this._trim,
int: this._int,
};
const headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
};
if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
headers['x-api-key'] = config.scrapingAnt.apiKey;
}
const driver = makeDriver(headers);
const xray = Xray({ filters });
xray.driver(driver);
this.xray = xray;
}
get x() {
return this.xray;
}
_removeNewline(value) {
return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
}
_trim(value) {
return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
}
_int(value) {
return typeof value === 'string' ? parseInt(value, 10) : value;
}
}
module.exports = new Scraper().x;
export default new Scraper().x;