diff --git a/index.js b/index.js
index d3dd128..5dd4b7f 100755
--- a/index.js
+++ b/index.js
@@ -10,6 +10,7 @@ import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/stor
import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
import logger from './lib/services/logger.js';
+import { bus } from './lib/services/events/event-bus.js';
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
const rawDir = config.sqlitepath || '/db';
@@ -45,29 +46,34 @@ ensureAdminUserExists();
ensureDemoUserExists();
await initTrackerCron();
-setInterval(
- (function exec() {
- const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
- if (!config.demoMode) {
- if (isDuringWorkingHoursOrNotSet) {
- config.lastRun = Date.now();
- jobStorage
- .getJobs()
- .filter((job) => job.enabled)
- .forEach((job) => {
- job.provider
- .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
- .forEach(async (prov) => {
- const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
- pro.init(prov, job.blacklist);
- await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
- });
- });
- } else {
- logger.debug('Working hours set. Skipping as outside of working hours.');
- }
+bus.on('jobs:runAll', () => {
+ logger.debug('Running Fredy Job manually');
+ execute();
+});
+
+const execute = () => {
+ const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+ if (!config.demoMode) {
+ if (isDuringWorkingHoursOrNotSet) {
+ config.lastRun = Date.now();
+ jobStorage
+ .getJobs()
+ .filter((job) => job.enabled)
+ .forEach((job) => {
+ job.provider
+ .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+ .forEach(async (prov) => {
+ const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+ pro.init(prov, job.blacklist);
+ await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+ });
+ });
+ } else {
+ logger.debug('Working hours set. Skipping as outside of working hours.');
}
- return exec;
- })(),
- INTERVAL,
-);
+ }
+};
+
+setInterval(execute, INTERVAL);
+//start once at startup
+execute();
diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js
index 5d47334..9cc4a2c 100644
--- a/lib/api/routes/jobRouter.js
+++ b/lib/api/routes/jobRouter.js
@@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js';
import { config } from '../../utils.js';
import { isAdmin } from '../security.js';
import logger from '../../services/logger.js';
+import { bus } from '../../services/events/event-bus.js';
+
const service = restana();
const jobRouter = service.newRouter();
+
function doesJobBelongsToUser(job, req) {
const userId = req.session.currentUser;
if (userId == null) {
@@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) {
}
return user.isAdmin || job.userId === user.id;
}
+
jobRouter.get('/', async (req, res) => {
const isUserAdmin = isAdmin(req);
//show only the jobs which belongs to the user (or all of the user is an admin)
@@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => {
};
res.send();
});
+
+jobRouter.post('/startAll', async (req, res) => {
+ bus.emit('jobs:runAll');
+ res.send();
+});
+
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
try {
diff --git a/lib/services/events/event-bus.js b/lib/services/events/event-bus.js
new file mode 100644
index 0000000..e56852b
--- /dev/null
+++ b/lib/services/events/event-bus.js
@@ -0,0 +1,2 @@
+import { EventEmitter } from 'node:events';
+export const bus = new EventEmitter();
diff --git a/lib/services/extractor/parser/parser.js b/lib/services/extractor/parser/parser.js
index 84b0e69..82d27f7 100644
--- a/lib/services/extractor/parser/parser.js
+++ b/lib/services/extractor/parser/parser.js
@@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
const result = [];
if ($(crawlContainer).length === 0) {
- logger.warn('No elements in crawl container found for url ', url);
+ logger.debug('No elements in crawl container found for url ', url);
return null;
}
diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js
index 0264464..d1b0075 100755
--- a/lib/services/storage/listingsStorage.js
+++ b/lib/services/storage/listingsStorage.js
@@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => {
SqliteConnection.withTransaction((db) => {
const stmt = db.prepare(
- `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
+ `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
link, created_at)
- VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
+ VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
@created_at)
- ON CONFLICT(hash) DO NOTHING`,
+ ON CONFLICT(job_id, hash) DO NOTHING`,
);
for (const item of listings) {
diff --git a/lib/services/storage/migrations/sql/2.new-indexes-for-listings.js b/lib/services/storage/migrations/sql/2.new-indexes-for-listings.js
new file mode 100644
index 0000000..19b2fac
--- /dev/null
+++ b/lib/services/storage/migrations/sql/2.new-indexes-for-listings.js
@@ -0,0 +1,10 @@
+// Migration: there needs to be a unique index on job_id and hash as only
+// this makes the listing indeed unique
+
+export function up(db) {
+ db.exec(`
+ DROP INDEX IF EXISTS idx_listings_hash;
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
+ ON listings (job_id, hash);
+ `);
+}
diff --git a/package.json b/package.json
index 50344b2..ea321dc 100755
--- a/package.json
+++ b/package.json
@@ -56,6 +56,7 @@
"Firefox ESR"
],
"dependencies": {
+ "@douyinfe/semi-icons": "^2.86.0",
"@douyinfe/semi-ui": "2.86.0",
"@sendgrid/mail": "8.1.5",
"@visactor/react-vchart": "^2.0.4",
diff --git a/ui/src/views/jobs/ProcessingTimes.jsx b/ui/src/views/jobs/ProcessingTimes.jsx
index a02acc5..9482433 100644
--- a/ui/src/views/jobs/ProcessingTimes.jsx
+++ b/ui/src/views/jobs/ProcessingTimes.jsx
@@ -1,6 +1,8 @@
import React from 'react';
import { format } from '../../services/time/timeService';
-import { Descriptions } from '@douyinfe/semi-ui';
+import { Button, Descriptions, Toast } from '@douyinfe/semi-ui';
+import { IconPlayCircle } from '@douyinfe/semi-icons';
+import { xhrPost } from '../../services/xhr.js';
export default function ProcessingTimes({ processingTimes = {} }) {
if (Object.keys(processingTimes).length === 0) {
@@ -24,6 +26,19 @@ export default function ProcessingTimes({ processingTimes = {} }) {
{format(processingTimes.lastRun + processingTimes.interval * 60000)}
+
+ }
+ aria-label="Start now"
+ onClick={async () => {
+ await xhrPost('/api/jobs/startAll', null);
+ Toast.success('Successfully triggered Fredy search.');
+ }}
+ >
+ Search now
+
+
>
)}
diff --git a/yarn.lock b/yarn.lock
index f746938..7c89705 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -973,7 +973,7 @@
remark-gfm "^4.0.0"
scroll-into-view-if-needed "^2.2.24"
-"@douyinfe/semi-icons@2.86.0":
+"@douyinfe/semi-icons@2.86.0", "@douyinfe/semi-icons@^2.86.0":
version "2.86.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.86.0.tgz#ee4355c81616ea4325627a3bb607ed9f9b9afac3"
integrity sha512-KEDlYYP1wdOqN28Ck0YcdCx7mSks8SRY4w4KKbXPaROzYNEyT2BRcJxwysMHfxL2IDfsroHrRPJsX9pnrmQqTg==