diff --git a/vripper-server/pom.xml b/vripper-server/pom.xml index 4a68b3f..56b7e30 100644 --- a/vripper-server/pom.xml +++ b/vripper-server/pom.xml @@ -59,7 +59,7 @@ net.sourceforge.htmlcleaner htmlcleaner - 2.22 + 2.24 io.projectreactor diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJobWrapper.java similarity index 95% rename from vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java rename to vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJobWrapper.java index 79521c7..89e03cc 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJobWrapper.java @@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.DataService; import java.time.LocalDateTime; @Slf4j -public class DownloadRunnable implements Runnable { +public class DownloadJobWrapper implements Runnable { private final DownloadService downloadService; private final DataService dataService; @@ -22,7 +22,7 @@ public class DownloadRunnable implements Runnable { private final DownloadJob downloadJob; - public DownloadRunnable(final DownloadJob downloadJob) { + public DownloadJobWrapper(final DownloadJob downloadJob) { downloadService = SpringContext.getBean(DownloadService.class); dataService = SpringContext.getBean(DataService.class); connectionService = SpringContext.getBean(ConnectionService.class); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java index 19fcca4..62b0953 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java @@ -11,10 +11,14 @@ import tn.mnlr.vripper.jpa.domain.enums.Status; import tn.mnlr.vripper.services.DataService; import tn.mnlr.vripper.services.PostService; import tn.mnlr.vripper.services.SettingsService; +import tn.mnlr.vripper.services.domain.Settings; import javax.annotation.PostConstruct; import java.util.*; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @Service @@ -27,41 +31,38 @@ public class DownloadService { private final ConcurrentHashMap threadCount = new ConcurrentHashMap<>(); private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE); - private final BlockingQueue executionQueue = new LinkedBlockingQueue<>(); - private final List executing = Collections.synchronizedList(new ArrayList<>()); + private final List running = Collections.synchronizedList(new ArrayList<>()); + private final List pending = new ArrayList<>(); - private final PendingQueue pendingQueue; - private final SettingsService settings; + private final SettingsService settingsService; private final DataService dataService; private final PostService postService; + private final List hosts; + private boolean pauseQ = false; - private Thread executionThread; private Thread pollThread; @Autowired public DownloadService( - PendingQueue pendingQueue, - SettingsService settings, + SettingsService settingsService, DataService dataService, - PostService postService) { - this.pendingQueue = pendingQueue; - this.settings = settings; + PostService postService, + List hosts) { + this.settingsService = settingsService; this.dataService = dataService; this.postService = postService; + this.hosts = hosts; } @PostConstruct private void init() { - executionThread = new Thread(this::start, "Executor thread"); - pollThread = new Thread(this::poll, "Polling thread"); + pollThread = new Thread(this::start, "Polling thread"); pollThread.start(); - executionThread.start(); } public void destroy() throws Exception { log.info("Shutting down ExecutionService"); - executionThread.interrupt(); pollThread.interrupt(); executor.shutdown(); dataService @@ -76,7 +77,7 @@ public class DownloadService { private void stopRunning(@NonNull String postId) { List stopping = new ArrayList<>(); - Iterator iterator = executing.iterator(); + Iterator iterator = running.iterator(); while (iterator.hasNext()) { DownloadJob downloadJob = iterator.next(); if (postId.equals(downloadJob.getPost().getPostId())) { @@ -125,18 +126,15 @@ public class DownloadService { post.setStatus(Status.PENDING); dataService.updatePostStatus(post.getStatus(), post.getId()); log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId)); - for (Image image : images) { - try { - pendingQueue.put(post, image); - } catch (InterruptedException e) { - log.warn("Thread was interrupted", e); - Thread.currentThread().interrupt(); - } - } + enqueue(post, images); } private boolean isPending(String postId) { - return pendingQueue.isPending(postId); + return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId)); + } + + private boolean isRunning(String postId) { + return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId)); } private void stop(String postId) { @@ -149,7 +147,7 @@ public class DownloadService { if (FINISHED.contains(post.getStatus())) { return; } - pendingQueue.stop(post); + pending.removeIf(p -> p.getPost().equals(post)); stopRunning(postId); dataService.stopImagesByPostIdAndIsNotCompleted(postId); dataService.finishPost(post); @@ -161,16 +159,12 @@ public class DownloadService { private boolean canRun(Host host) { boolean canRun; - AtomicInteger count = threadCount.get(host); - if (count == null) { - threadCount.put(host, new AtomicInteger(0)); - } + int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum(); canRun = - threadCount.get(host).get() < settings.getSettings().getMaxThreads() - && (settings.getSettings().getMaxTotalThreads() == 0 - ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE - : threadCount.values().stream().mapToInt(AtomicInteger::get).sum() - < settings.getSettings().getMaxTotalThreads()); + threadCount.get(host).get() < settingsService.getSettings().getMaxThreads() + && (settingsService.getSettings().getMaxTotalThreads() == 0 + ? totalRunning < MAX_POOL_SIZE + : totalRunning < settingsService.getSettings().getMaxTotalThreads()); if (canRun && !pauseQ) { threadCount.get(host).incrementAndGet(); return true; @@ -178,59 +172,102 @@ public class DownloadService { return false; } - private void poll() { - while (!Thread.interrupted()) { - try { - List peek = pendingQueue.peek(); - for (DownloadJob downloadJob : peek) { - if (canRun(downloadJob.getImage().getHost())) { - executionQueue.offer(downloadJob); - pendingQueue.remove(downloadJob); + private Map candidateCount() { + HashMap map = new HashMap<>(); + hosts.forEach( + h -> { + AtomicInteger count = threadCount.get(h); + if (count == null) { + count = new AtomicInteger(0); + threadCount.put(h, count); } - } - synchronized (threadCount) { - threadCount.wait(2_000); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + map.put(h, settingsService.getSettings().getMaxThreads() - count.get()); + }); + return map; + } + + private List getCandidates(Map candidateCount, int max) { + int i = 0; + List candidates = new ArrayList<>(); + for (DownloadJob downloadJob : pending) { + Host host = downloadJob.getImage().getHost(); + Integer maxPerHost = candidateCount.get(host); + if (maxPerHost > 0 && i < max) { + candidates.add(downloadJob); + candidateCount.put(host, maxPerHost + 1); + i++; + } + if (i >= max) { break; } } + return candidates; + } + + public void enqueue(Post post, Collection imageList) { + synchronized (this) { + for (Image image : imageList) { + log.debug(String.format("Enqueuing a job for %s", image.getUrl())); + image.init(); + dataService.updateImageStatus(image.getStatus(), image.getId()); + dataService.updateImageCurrent(image.getCurrent(), image.getId()); + DownloadJob downloadJob = + new DownloadJob(post, image, (Settings) settingsService.getSettings().clone()); + pending.add(downloadJob); + } + pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn())); + + this.notify(); + } } private void start() { while (!Thread.interrupted()) { try { - push(executionQueue.take()); + synchronized (this) { + List accepted = new ArrayList<>(); + List candidates = getCandidates(candidateCount(), MAX_POOL_SIZE); + candidates.forEach( + c -> { + if (canRun(c.getImage().getHost())) { + accepted.add(c); + } + }); + pending.removeAll(accepted); + accepted.forEach(this::push); + accepted.clear(); + this.wait(); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; - } catch (Exception e) { - log.error("Execution Service failed", e); - break; } } } private void push(DownloadJob downloadJob) { log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl())); - executor.execute(new DownloadRunnable(downloadJob)); - executing.add(downloadJob); + executor.execute(new DownloadJobWrapper(downloadJob)); + running.add(downloadJob); } - public synchronized void afterJobFinish(DownloadJob downloadJob) { - int count = pendingQueue.decrement(downloadJob.getPost().getPostId()); - if (count == 0) { - dataService.finishPost(downloadJob.getPost()); - } - threadCount.get(downloadJob.getImage().getHost()).decrementAndGet(); - executing.remove(downloadJob); - synchronized (threadCount) { - threadCount.notify(); + public void afterJobFinish(DownloadJob downloadJob) { + synchronized (this) { + running.remove(downloadJob); + threadCount.get(downloadJob.getImage().getHost()).decrementAndGet(); + if (!isPending(downloadJob.getPost().getPostId()) + && !isRunning(downloadJob.getPost().getPostId())) { + dataService.finishPost(downloadJob.getPost()); + } + this.notify(); } } + public int pendingCount() { + return pending.size(); + } + public int runningCount() { - return executing.size(); + return running.size(); } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java deleted file mode 100644 index 8d74acd..0000000 --- a/vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java +++ /dev/null @@ -1,118 +0,0 @@ -package tn.mnlr.vripper.download; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import tn.mnlr.vripper.host.Host; -import tn.mnlr.vripper.jpa.domain.Image; -import tn.mnlr.vripper.jpa.domain.Post; -import tn.mnlr.vripper.services.DataService; -import tn.mnlr.vripper.services.SettingsService; -import tn.mnlr.vripper.services.domain.Settings; - -import javax.annotation.PostConstruct; -import java.util.*; -import java.util.concurrent.BlockingDeque; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Predicate; - -@Service -@Slf4j -public class PendingQueue { - - private final DataService dataService; - private final SettingsService settingsService; - private final List hosts; - - private final ConcurrentHashMap> pendingQ = - new ConcurrentHashMap<>(); - private final ConcurrentHashMap toBeExecuted = new ConcurrentHashMap<>(); - - @Autowired - public PendingQueue(DataService dataService, SettingsService settingsService, List hosts) { - this.dataService = dataService; - this.settingsService = settingsService; - this.hosts = hosts; - } - - @PostConstruct - private void init() { - hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>())); - } - - public void put(Post post, Image image) throws InterruptedException { - log.debug(String.format("Enqueuing a job for %s", image.getUrl())); - image.init(); - dataService.updateImageStatus(image.getStatus(), image.getId()); - dataService.updateImageCurrent(image.getCurrent(), image.getId()); - DownloadJob downloadJob = - new DownloadJob(post, image, (Settings) settingsService.getSettings().clone()); - pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob); - checkKey(post.getPostId()); - toBeExecuted.get(post.getPostId()).incrementAndGet(); - } - - private synchronized void checkKey(String postId) { - if (!toBeExecuted.containsKey(postId)) { - toBeExecuted.put(postId, new AtomicInteger(0)); - } - } - - public void remove(final DownloadJob downloadJob) { - pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob); - } - - public List peek() { - List downloadJobs = new ArrayList<>(); - if (hosts.size() == 0) { - return downloadJobs; - } - for (Host host : hosts) { - Iterator it = pendingQ.get(host).iterator(); - for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) { - DownloadJob downloadJob = it.hasNext() ? it.next() : null; - if (downloadJob != null) { - downloadJobs.add(downloadJob); - } - } - } - return downloadJobs; - } - - public void enqueue(Post post, Set images) throws InterruptedException { - for (Image image : images) { - put(post, image); - } - } - - public int size() { - return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum(); - } - - public void stop(Post post) { - Predicate predicate = next -> next.getImage().getPostId().equals(post.getPostId()); - for (Map.Entry> entry : pendingQ.entrySet()) { - entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId())); - entry.getValue().removeIf(predicate); - decrement(post.getPostId()); - } - } - - public boolean isPending(String postId) { - return toBeExecuted.containsKey(postId); - } - - public synchronized int decrement(String postId) { - AtomicInteger counter = toBeExecuted.get(postId); - if (counter == null) { - return 0; - } - int count = counter.decrementAndGet(); - if (count == 0) { - toBeExecuted.remove(postId); - } - return count; - } -} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java b/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java index 494cd9f..985324d 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java @@ -28,6 +28,7 @@ public class Event { LOG_EVENT_REMOVE, VG_USER, GLOBAL_STATE, - BYTES_PER_SECOND + BYTES_PER_SECOND, + SETTINGS_UPDATE } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java index 1dd4be4..4f26b0c 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java @@ -7,6 +7,7 @@ import lombok.Setter; import lombok.ToString; import tn.mnlr.vripper.jpa.domain.enums.Status; +import java.time.LocalDateTime; import java.util.Collections; import java.util.Objects; import java.util.Set; @@ -49,6 +50,8 @@ public class Post { private Metadata metadata; + private LocalDateTime addedOn; + public Post( String title, String url, @@ -65,6 +68,7 @@ public class Post { this.threadTitle = threadTitle; this.securityToken = securityToken; status = Status.STOPPED; + addedOn = LocalDateTime.now(); } @Override diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java index 821fe04..7269c8e 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java @@ -13,6 +13,7 @@ import tn.mnlr.vripper.jpa.repositories.IPostRepository; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.List; import java.util.Optional; import java.util.Set; @@ -37,7 +38,7 @@ public class PostRepository implements IPostRepository { public Post save(Post post) { long id = nextId(); jdbcTemplate.update( - "INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", id, post.getDone(), post.getForum(), @@ -52,7 +53,8 @@ public class PostRepository implements IPostRepository { post.getThreadTitle(), post.getTitle(), post.getTotal(), - post.getUrl()); + post.getUrl(), + Timestamp.valueOf(post.getAddedOn())); post.setId(id); eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id)); return post; @@ -198,6 +200,7 @@ class PostRowMapper implements RowMapper { if ((previews = rs.getString("post.PREVIEWS")) != null) { post.setPreviews(Set.of(previews.split(DELIMITER))); } + post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime()); Long metadataId = rs.getLong("metadata.POST_ID_REF"); if (!rs.wasNull()) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java b/vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java deleted file mode 100644 index c3e1bb0..0000000 --- a/vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java +++ /dev/null @@ -1,7 +0,0 @@ -package tn.mnlr.vripper.listener; - -import reactor.core.publisher.Flux; - -public interface DataEventListener { - Flux getDataFlux(); -} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java b/vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java deleted file mode 100644 index b17b6ed..0000000 --- a/vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java +++ /dev/null @@ -1,7 +0,0 @@ -package tn.mnlr.vripper.listener; - -import reactor.core.publisher.Sinks.EmitFailureHandler; - -public class EmitHandler { - public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true; -} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java index 630687a..c4f9952 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java @@ -17,8 +17,9 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import reactor.core.Disposable; -import reactor.core.publisher.Flux; import tn.mnlr.vripper.download.DownloadJob; +import tn.mnlr.vripper.event.Event; +import tn.mnlr.vripper.event.EventBus; import tn.mnlr.vripper.services.domain.Settings; import javax.annotation.PreDestroy; @@ -41,22 +42,25 @@ public class ConnectionService { private int connectionTimeout; private int maxAttempts; - public ConnectionService(SettingsService settingsService) { + public ConnectionService(EventBus eventBus, SettingsService settingsService) { connectionTimeout = settingsService.getSettings().getConnectionTimeout(); maxAttempts = settingsService.getSettings().getMaxAttempts(); - Flux settingsFlux = settingsService.getSettingsFlux(); disposable = - settingsFlux.subscribe( - settings -> { - if (connectionTimeout != settings.getConnectionTimeout()) { - connectionTimeout = settings.getConnectionTimeout(); - buildRequestConfig(); - } - if (maxAttempts != settings.getMaxAttempts()) { - maxAttempts = settings.getMaxAttempts(); - buildRetryPolicy(); - } - }); + eventBus + .flux() + .filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE)) + .map(e -> ((Settings) e.getData())) + .subscribe( + settings -> { + if (connectionTimeout != settings.getConnectionTimeout()) { + connectionTimeout = settings.getConnectionTimeout(); + buildRequestConfig(); + } + if (maxAttempts != settings.getMaxAttempts()) { + maxAttempts = settings.getMaxAttempts(); + buildRetryPolicy(); + } + }); buildRequestConfig(); buildRetryPolicy(); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java index 95332df..73aef1e 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java @@ -6,7 +6,6 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import tn.mnlr.vripper.download.DownloadService; -import tn.mnlr.vripper.download.PendingQueue; import tn.mnlr.vripper.event.Event; import tn.mnlr.vripper.event.EventBus; import tn.mnlr.vripper.services.domain.GlobalState; @@ -15,7 +14,6 @@ import tn.mnlr.vripper.services.domain.GlobalState; @EnableScheduling public class GlobalStateService { - private final PendingQueue pendingQueue; private final DownloadService downloadService; private final DataService dataService; private final EventBus eventBus; @@ -23,11 +21,7 @@ public class GlobalStateService { @Autowired public GlobalStateService( - PendingQueue pendingQueue, - DownloadService downloadService, - DataService dataService, - EventBus eventBus) { - this.pendingQueue = pendingQueue; + DownloadService downloadService, DataService dataService, EventBus eventBus) { this.downloadService = downloadService; this.dataService = dataService; this.eventBus = eventBus; @@ -37,7 +31,9 @@ public class GlobalStateService { private void interval() { GlobalState newGlobalState = new GlobalState( - downloadService.runningCount(), pendingQueue.size(), dataService.countErrorImages()); + downloadService.runningCount(), + downloadService.pendingCount(), + dataService.countErrorImages()); if (!newGlobalState.equals(currentState)) { currentState = newGlobalState; eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState)); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java index fa3219a..18533cf 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java @@ -12,12 +12,16 @@ import org.apache.http.util.EntityUtils; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Node; +import reactor.core.Disposable; +import tn.mnlr.vripper.event.Event; +import tn.mnlr.vripper.event.EventBus; import tn.mnlr.vripper.exception.DownloadException; import tn.mnlr.vripper.exception.PostParseException; import tn.mnlr.vripper.jpa.domain.Metadata; import tn.mnlr.vripper.jpa.domain.Post; -import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable; +import tn.mnlr.vripper.tasks.MetadataRunnable; +import javax.annotation.PreDestroy; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -38,19 +42,33 @@ public class MetadataService { private final XpathService xpathService; private final Map fetchingMetadata = new ConcurrentHashMap<>(); private final ThreadPoolService threadPoolService; + private final Disposable disposable; public MetadataService( ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService, - ThreadPoolService threadPoolService) { + ThreadPoolService threadPoolService, + EventBus eventBus) { this.cm = cm; this.VGAuthService = VGAuthService; this.htmlProcessorService = htmlProcessorService; this.xpathService = xpathService; this.threadPoolService = threadPoolService; cache = Caffeine.newBuilder().expireAfterWrite(30, TimeUnit.MINUTES).build(this::fetchMetadata); + disposable = + eventBus + .flux() + .filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE)) + .subscribe(e -> this.cache.invalidateAll()); + } + + @PreDestroy + private void destroy() { + if (disposable != null) { + disposable.dispose(); + } } public Metadata get(Post post) throws ExecutionException { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java index 1b26aa8..ce00c34 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java @@ -5,13 +5,17 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import reactor.core.Disposable; +import tn.mnlr.vripper.event.Event; +import tn.mnlr.vripper.event.EventBus; import tn.mnlr.vripper.jpa.domain.Post; import tn.mnlr.vripper.jpa.domain.Queued; import tn.mnlr.vripper.services.domain.MultiPostScanParser; import tn.mnlr.vripper.services.domain.MultiPostScanResult; -import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable; -import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable; +import tn.mnlr.vripper.tasks.AddPostRunnable; +import tn.mnlr.vripper.tasks.AddQueuedRunnable; +import javax.annotation.PreDestroy; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -24,12 +28,14 @@ public class PostService { private final ThreadPoolService threadPoolService; private final MetadataService metadataService; private final LoadingCache cache; + private final Disposable disposable; @Autowired public PostService( DataService dataService, ThreadPoolService threadPoolService, - MetadataService metadataService) { + MetadataService metadataService, + EventBus eventBus) { this.dataService = dataService; this.threadPoolService = threadPoolService; this.metadataService = metadataService; @@ -37,6 +43,18 @@ public class PostService { Caffeine.newBuilder() .expireAfterWrite(5, TimeUnit.MINUTES) .build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse()); + disposable = + eventBus + .flux() + .filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE)) + .subscribe(e -> this.cache.invalidateAll()); + } + + @PreDestroy + private void destroy() { + if (disposable != null) { + disposable.dispose(); + } } public void stopFetchingMetadata(Post post) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java index bcc5256..25a3c7e 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java @@ -12,11 +12,10 @@ import org.apache.commons.codec.digest.DigestUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Sinks; import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.event.Event; +import tn.mnlr.vripper.event.EventBus; import tn.mnlr.vripper.exception.ValidationException; -import tn.mnlr.vripper.listener.EmitHandler; import tn.mnlr.vripper.services.domain.Settings; import javax.annotation.PostConstruct; @@ -46,7 +45,7 @@ public class SettingsService { private final Set proxies = new HashSet<>(); - private Sinks.Many sink = Sinks.many().multicast().onBackpressureBuffer(); + private final EventBus eventBus; @Getter private Settings settings = new Settings(); @@ -54,7 +53,10 @@ public class SettingsService { private Resource defaultProxies; public SettingsService( - @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) { + @Value("${base.dir}") String baseDir, + @Value("${base.dir.name}") String baseDirName, + EventBus eventBus) { + this.eventBus = eventBus; this.configPath = Paths.get(baseDir, baseDirName, "config.json"); this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json"); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -64,7 +66,7 @@ public class SettingsService { private void init() { loadViperProxies(); restore(); - sink.emitNext(settings, EmitHandler.RETRY); + eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings)); } private void loadViperProxies() { @@ -93,10 +95,6 @@ public class SettingsService { proxies.add("https://vipergirls.to"); } - Flux getSettingsFlux() { - return sink.asFlux(); - } - public List getProxies() { return new ArrayList<>(proxies); } @@ -116,7 +114,7 @@ public class SettingsService { this.settings = settings; save(); - sink.emitNext(settings, EmitHandler.RETRY); + eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings)); } public void restore() { @@ -232,7 +230,6 @@ public class SettingsService { @PreDestroy private void destroy() { save(); - sink.emitComplete(EmitHandler.RETRY); } public void check(Settings settings) throws ValidationException { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java index de5523b..b6ddc67 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java @@ -19,7 +19,7 @@ import tn.mnlr.vripper.event.Event; import tn.mnlr.vripper.event.EventBus; import tn.mnlr.vripper.exception.VripperException; import tn.mnlr.vripper.jpa.domain.Post; -import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable; +import tn.mnlr.vripper.tasks.LeaveThanksRunnable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -51,8 +51,12 @@ public class VGAuthService { this.cm = cm; this.settingsService = settingsService; this.threadPoolService = threadPoolService; - disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate()); this.eventBus = eventBus; + disposable = + eventBus + .flux() + .filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE)) + .subscribe(e -> authenticate()); } @PostConstruct diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java index ba16a60..9cd1ba6 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java @@ -68,8 +68,6 @@ public class MultiPostScanParser { } try { - // - // System.out.println(EntityUtils.toString(response.getEntity())); factory .newSAXParser() .parse( diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddPostRunnable.java similarity index 92% rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddPostRunnable.java index daacf2e..2d32649 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddPostRunnable.java @@ -1,9 +1,9 @@ -package tn.mnlr.vripper.services.domain.tasks; +package tn.mnlr.vripper.tasks; import lombok.extern.slf4j.Slf4j; import tn.mnlr.vripper.SpringContext; import tn.mnlr.vripper.Utils; -import tn.mnlr.vripper.download.PendingQueue; +import tn.mnlr.vripper.download.DownloadService; import tn.mnlr.vripper.exception.PostParseException; import tn.mnlr.vripper.jpa.domain.Image; import tn.mnlr.vripper.jpa.domain.LogEvent; @@ -31,11 +31,11 @@ public class AddPostRunnable implements Runnable { private final DataService dataService; private final MetadataService metadataService; private final SettingsService settingsService; - private final PendingQueue pendingQueue; private final VGAuthService VGAuthService; private final LogEvent logEvent; private final ILogEventRepository eventRepository; private final String link; + private final DownloadService downloadService; public AddPostRunnable(String postId, String threadId) { this.postId = postId; @@ -43,7 +43,7 @@ public class AddPostRunnable implements Runnable { this.dataService = SpringContext.getBean(DataService.class); this.metadataService = SpringContext.getBean(MetadataService.class); this.settingsService = SpringContext.getBean(SettingsService.class); - this.pendingQueue = SpringContext.getBean(PendingQueue.class); + this.downloadService = SpringContext.getBean(DownloadService.class); this.VGAuthService = SpringContext.getBean(VGAuthService.class); this.eventRepository = SpringContext.getBean(ILogEventRepository.class); link = @@ -112,13 +112,7 @@ public class AddPostRunnable implements Runnable { if (settingsService.getSettings().getAutoStart()) { log.debug("Auto start downloads option is enabled"); post.setStatus(Status.PENDING); - try { - pendingQueue.enqueue(post, images); - } catch (InterruptedException e) { - log.warn("Interruption was caught"); - Thread.currentThread().interrupt(); - return; - } + downloadService.enqueue(post, images); log.debug(String.format("Done enqueuing jobs for %s", post.getUrl())); } else { post.setStatus(Status.STOPPED); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddQueuedRunnable.java similarity index 98% rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddQueuedRunnable.java index 9d95841..daff6e5 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddQueuedRunnable.java @@ -1,4 +1,4 @@ -package tn.mnlr.vripper.services.domain.tasks; +package tn.mnlr.vripper.tasks; import lombok.extern.slf4j.Slf4j; import tn.mnlr.vripper.SpringContext; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LeaveThanksRunnable.java similarity index 99% rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/LeaveThanksRunnable.java index 489a329..9e8515d 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LeaveThanksRunnable.java @@ -1,4 +1,4 @@ -package tn.mnlr.vripper.services.domain.tasks; +package tn.mnlr.vripper.tasks; import lombok.extern.slf4j.Slf4j; import org.apache.http.NameValuePair; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LinkScanRunnable.java similarity index 98% rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/LinkScanRunnable.java index acbba58..8b14f31 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LinkScanRunnable.java @@ -1,4 +1,4 @@ -package tn.mnlr.vripper.services.domain.tasks; +package tn.mnlr.vripper.tasks; import lombok.extern.slf4j.Slf4j; import org.springframework.lang.NonNull; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/MetadataRunnable.java similarity index 98% rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/MetadataRunnable.java index aa68195..cc578c8 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/MetadataRunnable.java @@ -1,4 +1,4 @@ -package tn.mnlr.vripper.services.domain.tasks; +package tn.mnlr.vripper.tasks; import lombok.Getter; import lombok.NonNull; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java index 16580f3..ad14ea2 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java @@ -12,8 +12,8 @@ import tn.mnlr.vripper.jpa.domain.Queued; import tn.mnlr.vripper.services.*; import tn.mnlr.vripper.services.domain.MultiPostItem; import tn.mnlr.vripper.services.domain.MultiPostScanResult; -import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable; -import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable; +import tn.mnlr.vripper.tasks.AddPostRunnable; +import tn.mnlr.vripper.tasks.LinkScanRunnable; import tn.mnlr.vripper.web.restendpoints.domain.*; import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException; import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException; diff --git a/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml b/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml index a633bac..63cdb32 100644 --- a/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml +++ b/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml @@ -161,4 +161,11 @@ initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION" referencedColumnNames="ID" referencedTableName="POST"/> + + + + + + +