Puppeteer rewrite (#119)

* Moving to puppeteer | removing scrapingAnt
This commit is contained in:
Christian Kellner
2024-12-17 12:38:28 +01:00
committed by GitHub
parent 58965a6f1b
commit 214e714c03
30 changed files with 1367 additions and 1190 deletions

View File

@@ -5,7 +5,6 @@ import * as userStorage from '../../services/storage/userStorage.js';
import * as immoscoutProvider from '../../provider/immoscout.js';
import { config } from '../../utils.js';
import { isAdmin } from '../security.js';
import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
import {trackDemoJobCreated} from '../../services/tracking/Tracker.js';
const service = restana();
const jobRouter = service.newRouter();
@@ -27,34 +26,14 @@ jobRouter.get('/', async (req, res) => {
res.send();
});
jobRouter.get('/processingTimes', async (req, res) => {
let scrapingAntData = {};
if (isScrapingAntApiKeySet()) {
try {
const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
scrapingAntData = await response.json();
} catch (Exception) {
console.error('Could not query plan data from scraping ant.', Exception);
}
}
res.body = {
interval: config.interval,
lastRun: config.lastRun || null,
scrapingAntData,
error: scrapingAntData?.detail == null ? null : scrapingAntData?.detail
lastRun: config.lastRun || null
};
res.send();
});
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
if (
provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
(config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
) {
res.send(
new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
);
return;
}
try {
jobStorage.upsertJob({
userId: req.session.currentUser,