External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding new unique index, adding button to start now

Browse Source
This commit is contained in:
orangecoding
2025-09-18 20:48:25 +02:00
parent 4f79c5cba2
commit d1dad7fd3b
9 changed files with 75 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/services/events/event-bus.js Normal file
View File

@@ -0,0 +1,2 @@
import { EventEmitter } from 'node:events';
export const bus = new EventEmitter();

2
lib/services/extractor/parser/parser.js
View File

@@ -21,7 +21,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
const result = [];
if ($(crawlContainer).length === 0) {
logger.warn('No elements in crawl container found for url ', url);
logger.debug('No elements in crawl container found for url ', url);
return null;
}

6
lib/services/storage/listingsStorage.js
View File

@@ -85,11 +85,11 @@ export const storeListings = (jobId, providerId, listings) => {
SqliteConnection.withTransaction((db) => {
const stmt = db.prepare(
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, city,
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
link, created_at)
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @city, @link,
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
@created_at)
ON CONFLICT(hash) DO NOTHING`,
ON CONFLICT(job_id, hash) DO NOTHING`,
);
for (const item of listings) {

10
lib/services/storage/migrations/sql/2.new-indexes-for-listings.js Normal file
View File

@@ -0,0 +1,10 @@
// Migration: there needs to be a unique index on job_id and hash as only
// this makes the listing indeed unique
export function up(db) {
db.exec(`
DROP INDEX IF EXISTS idx_listings_hash;
CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_job_hash
ON listings (job_id, hash);
`);
}