diff --git a/index.js b/index.js index d3dd128..5dd4b7f 100755 --- a/index.js +++ b/index.js @@ -10,6 +10,7 @@ import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/stor import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js'; import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js'; import logger from './lib/services/logger.js'; +import { bus } from './lib/services/events/event-bus.js'; // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath) const rawDir = config.sqlitepath || '/db'; @@ -45,29 +46,34 @@ ensureAdminUserExists(); ensureDemoUserExists(); await initTrackerCron(); -setInterval( - (function exec() { - const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now()); - if (!config.demoMode) { - if (isDuringWorkingHoursOrNotSet) { - config.lastRun = Date.now(); - jobStorage - .getJobs() - .filter((job) => job.enabled) - .forEach((job) => { - job.provider - .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null) - .forEach(async (prov) => { - const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id); - pro.init(prov, job.blacklist); - await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute(); - }); - }); - } else { - logger.debug('Working hours set. Skipping as outside of working hours.'); - } +bus.on('jobs:runAll', () => { + logger.debug('Running Fredy Job manually'); + execute(); +}); + +const execute = () => { + const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now()); + if (!config.demoMode) { + if (isDuringWorkingHoursOrNotSet) { + config.lastRun = Date.now(); + jobStorage + .getJobs() + .filter((job) => job.enabled) + .forEach((job) => { + job.provider + .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null) + .forEach(async (prov) => { + const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id); + pro.init(prov, job.blacklist); + await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute(); + }); + }); + } else { + logger.debug('Working hours set. Skipping as outside of working hours.'); } - return exec; - })(), - INTERVAL, -); + } +}; + +setInterval(execute, INTERVAL); +//start once at startup +execute(); diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js index 5d47334..9cc4a2c 100644 --- a/lib/api/routes/jobRouter.js +++ b/lib/api/routes/jobRouter.js @@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js'; import { config } from '../../utils.js'; import { isAdmin } from '../security.js'; import logger from '../../services/logger.js'; +import { bus } from '../../services/events/event-bus.js'; + const service = restana(); const jobRouter = service.newRouter(); + function doesJobBelongsToUser(job, req) { const userId = req.session.currentUser; if (userId == null) { @@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) { } return user.isAdmin || job.userId === user.id; } + jobRouter.get('/', async (req, res) => { const isUserAdmin = isAdmin(req); //show only the jobs which belongs to the user (or all of the user is an admin) @@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => { }; res.send(); }); + +jobRouter.post('/startAll', async (req, res) => { + bus.emit('jobs:runAll'); + res.send(); +}); + jobRouter.post('/', async (req, res) => { const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body; try { diff --git a/lib/services/events/event-bus.js b/lib/services/events/event-bus.js new file mode 100644 index 0000000..e56852b --- /dev/null +++ b/lib/services/events/event-bus.js @@ -0,0 +1,2 @@ +import { EventEmitter } from 'node:events'; +export const bus = new EventEmitter(); diff --git a/lib/services/extractor/parser/parser.js b/lib/services/extractor/parser/parser.js index 84b0e69..82d27f7 100644 --- a/lib/services/extractor/parser/parser.js +++ b/lib/services/extractor/parser/parser.js @@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) { const result = []; if ($(crawlContainer).length === 0) { - logger.warn('No elements in crawl container found for url ', url); + logger.debug('No elements in crawl container found for url ', url); return null; } diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index 0264464..d1b0075 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => { SqliteConnection.withTransaction((db) => { const stmt = db.prepare( - `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city, + `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, link, created_at) - VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link, + VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link, @created_at) - ON CONFLICT(hash) DO NOTHING`, + ON CONFLICT(job_id, hash) DO NOTHING`, ); for (const item of listings) { diff --git a/lib/services/storage/migrations/sql/2.new-indexes-for-listings.js b/lib/services/storage/migrations/sql/2.new-indexes-for-listings.js new file mode 100644 index 0000000..19b2fac --- /dev/null +++ b/lib/services/storage/migrations/sql/2.new-indexes-for-listings.js @@ -0,0 +1,10 @@ +// Migration: there needs to be a unique index on job_id and hash as only +// this makes the listing indeed unique + +export function up(db) { + db.exec(` + DROP INDEX IF EXISTS idx_listings_hash; + CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash + ON listings (job_id, hash); + `); +} diff --git a/package.json b/package.json index 50344b2..ea321dc 100755 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "Firefox ESR" ], "dependencies": { + "@douyinfe/semi-icons": "^2.86.0", "@douyinfe/semi-ui": "2.86.0", "@sendgrid/mail": "8.1.5", "@visactor/react-vchart": "^2.0.4", diff --git a/ui/src/views/jobs/ProcessingTimes.jsx b/ui/src/views/jobs/ProcessingTimes.jsx index a02acc5..9482433 100644 --- a/ui/src/views/jobs/ProcessingTimes.jsx +++ b/ui/src/views/jobs/ProcessingTimes.jsx @@ -1,6 +1,8 @@ import React from 'react'; import { format } from '../../services/time/timeService'; -import { Descriptions } from '@douyinfe/semi-ui'; +import { Button, Descriptions, Toast } from '@douyinfe/semi-ui'; +import { IconPlayCircle } from '@douyinfe/semi-icons'; +import { xhrPost } from '../../services/xhr.js'; export default function ProcessingTimes({ processingTimes = {} }) { if (Object.keys(processingTimes).length === 0) { @@ -24,6 +26,19 @@ export default function ProcessingTimes({ processingTimes = {} }) { {format(processingTimes.lastRun + processingTimes.interval * 60000)} + + + )} diff --git a/yarn.lock b/yarn.lock index f746938..7c89705 100644 --- a/yarn.lock +++ b/yarn.lock @@ -973,7 +973,7 @@ remark-gfm "^4.0.0" scroll-into-view-if-needed "^2.2.24" -"@douyinfe/semi-icons@2.86.0": +"@douyinfe/semi-icons@2.86.0", "@douyinfe/semi-icons@^2.86.0": version "2.86.0" resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.86.0.tgz#ee4355c81616ea4325627a3bb607ed9f9b9afac3" integrity sha512-KEDlYYP1wdOqN28Ck0YcdCx7mSks8SRY4w4KKbXPaROzYNEyT2BRcJxwysMHfxL2IDfsroHrRPJsX9pnrmQqTg==