adding new unique index, adding button to start now

index.js

@@ -10,6 +10,7 @@ import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/stor
 import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
 import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
 import logger from './lib/services/logger.js';
+import { bus } from './lib/services/events/event-bus.js';
 
 // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
 const rawDir = config.sqlitepath || '/db';
@@ -45,29 +46,34 @@ ensureAdminUserExists();
 ensureDemoUserExists();
 await initTrackerCron();
 
-setInterval(
-  (function exec() {
-    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-    if (!config.demoMode) {
-      if (isDuringWorkingHoursOrNotSet) {
-        config.lastRun = Date.now();
-        jobStorage
-          .getJobs()
-          .filter((job) => job.enabled)
-          .forEach((job) => {
-            job.provider
-              .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-              .forEach(async (prov) => {
-                const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-                pro.init(prov, job.blacklist);
-                await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
-              });
-          });
-      } else {
-        logger.debug('Working hours set. Skipping as outside of working hours.');
-      }
+bus.on('jobs:runAll', () => {
+  logger.debug('Running Fredy Job manually');
+  execute();
+});
+
+const execute = () => {
+  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+  if (!config.demoMode) {
+    if (isDuringWorkingHoursOrNotSet) {
+      config.lastRun = Date.now();
+      jobStorage
+        .getJobs()
+        .filter((job) => job.enabled)
+        .forEach((job) => {
+          job.provider
+            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+            .forEach(async (prov) => {
+              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+              pro.init(prov, job.blacklist);
+              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+            });
+        });
+    } else {
+      logger.debug('Working hours set. Skipping as outside of working hours.');
     }
-    return exec;
-  })(),
-  INTERVAL,
-);
+  }
+};
+
+setInterval(execute, INTERVAL);
+//start once at startup
+execute();

lib/api/routes/jobRouter.js

@@ -4,8 +4,11 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import logger from '../../services/logger.js';
+import { bus } from '../../services/events/event-bus.js';
+
 const service = restana();
 const jobRouter = service.newRouter();
+
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -17,6 +20,7 @@ function doesJobBelongsToUser(job, req) {
   }
   return user.isAdmin || job.userId === user.id;
 }
+
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
   //show only the jobs which belongs to the user (or all of the user is an admin)
@@ -30,6 +34,12 @@ jobRouter.get('/processingTimes', async (req, res) => {
   };
   res.send();
 });
+
+jobRouter.post('/startAll', async (req, res) => {
+  bus.emit('jobs:runAll');
+  res.send();
+});
+
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
   try {

lib/services/events/event-bus.js

@@ -0,0 +1,2 @@
+import { EventEmitter } from 'node:events';
+export const bus = new EventEmitter();

lib/services/extractor/parser/parser.js

@@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
   const result = [];
 
   if ($(crawlContainer).length === 0) {
-    logger.warn('No elements in crawl container found for url ', url);
+    logger.debug('No elements in crawl container found for url ', url);
     return null;
   }
 

lib/services/storage/listingsStorage.js

@@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => {
 
   SqliteConnection.withTransaction((db) => {
     const stmt = db.prepare(
-      `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
+      `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
                              link, created_at)
-       VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
+       VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
                @created_at)
-       ON CONFLICT(hash) DO NOTHING`,
+       ON CONFLICT(job_id, hash) DO NOTHING`,
     );
 
     for (const item of listings) {

lib/services/storage/migrations/sql/2.new-indexes-for-listings.js

@@ -0,0 +1,10 @@
+// Migration: there needs to be a unique index on job_id and hash as only
+// this makes the listing indeed unique
+
+export function up(db) {
+  db.exec(`
+    DROP INDEX IF EXISTS idx_listings_hash;
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
+      ON listings (job_id, hash);
+  `);
+}

package.json

@@ -56,6 +56,7 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@douyinfe/semi-icons": "^2.86.0",
     "@douyinfe/semi-ui": "2.86.0",
     "@sendgrid/mail": "8.1.5",
     "@visactor/react-vchart": "^2.0.4",

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,6 +1,8 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import { Descriptions } from '@douyinfe/semi-ui';
+import { Button, Descriptions, Toast } from '@douyinfe/semi-ui';
+import { IconPlayCircle } from '@douyinfe/semi-icons';
+import { xhrPost } from '../../services/xhr.js';
 
 export default function ProcessingTimes({ processingTimes = {} }) {
   if (Object.keys(processingTimes).length === 0) {
@@ -24,6 +26,19 @@ export default function ProcessingTimes({ processingTimes = {} }) {
             <Descriptions.Item itemKey="Next run">
               {format(processingTimes.lastRun + processingTimes.interval * 60000)}
             </Descriptions.Item>
+            <Descriptions.Item itemKey="Find Listings now">
+              <Button
+                size="small"
+                icon={<IconPlayCircle />}
+                aria-label="Start now"
+                onClick={async () => {
+                  await xhrPost('/api/jobs/startAll', null);
+                  Toast.success('Successfully triggered Fredy search.');
+                }}
+              >
+                Search now
+              </Button>
+            </Descriptions.Item>
           </>
         )}
       </Descriptions>

yarn.lock

@@ -973,7 +973,7 @@
     remark-gfm "^4.0.0"
     scroll-into-view-if-needed "^2.2.24"
 
-"@douyinfe/semi-icons@2.86.0":
+"@douyinfe/semi-icons@2.86.0", "@douyinfe/semi-icons@^2.86.0":
   version "2.86.0"
   resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.86.0.tgz#ee4355c81616ea4325627a3bb607ed9f9b9afac3"
   integrity sha512-KEDlYYP1wdOqN28Ck0YcdCx7mSks8SRY4w4KKbXPaROzYNEyT2BRcJxwysMHfxL2IDfsroHrRPJsX9pnrmQqTg==