From 53fd14935b4871e7cc5ab447d6955c2007542774 Mon Sep 17 00:00:00 2001
From: death-claw <53543762+death-claw@users.noreply.github.com>
Date: Sun, 9 May 2021 22:39:22 +0100
Subject: [PATCH] Enhance download queue logic
---
vripper-server/pom.xml | 2 +-
...dRunnable.java => DownloadJobWrapper.java} | 4 +-
.../vripper/download/DownloadService.java | 167 +++++++++++-------
.../mnlr/vripper/download/PendingQueue.java | 118 -------------
.../java/tn/mnlr/vripper/event/Event.java | 3 +-
.../java/tn/mnlr/vripper/jpa/domain/Post.java | 4 +
.../jpa/repositories/impl/PostRepository.java | 7 +-
.../vripper/listener/DataEventListener.java | 7 -
.../tn/mnlr/vripper/listener/EmitHandler.java | 7 -
.../vripper/services/ConnectionService.java | 32 ++--
.../vripper/services/GlobalStateService.java | 12 +-
.../vripper/services/MetadataService.java | 22 ++-
.../tn/mnlr/vripper/services/PostService.java | 24 ++-
.../vripper/services/SettingsService.java | 21 +--
.../mnlr/vripper/services/VGAuthService.java | 8 +-
.../services/domain/MultiPostScanParser.java | 2 -
.../domain => }/tasks/AddPostRunnable.java | 16 +-
.../domain => }/tasks/AddQueuedRunnable.java | 2 +-
.../tasks/LeaveThanksRunnable.java | 2 +-
.../domain => }/tasks/LinkScanRunnable.java | 2 +-
.../domain => }/tasks/MetadataRunnable.java | 2 +-
.../web/restendpoints/PostRestEndpoint.java | 4 +-
.../db/changelog/db.changelog-master.xml | 7 +
23 files changed, 212 insertions(+), 263 deletions(-)
rename vripper-server/src/main/java/tn/mnlr/vripper/download/{DownloadRunnable.java => DownloadJobWrapper.java} (95%)
delete mode 100644 vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java
delete mode 100644 vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java
delete mode 100644 vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java
rename vripper-server/src/main/java/tn/mnlr/vripper/{services/domain => }/tasks/AddPostRunnable.java (92%)
rename vripper-server/src/main/java/tn/mnlr/vripper/{services/domain => }/tasks/AddQueuedRunnable.java (98%)
rename vripper-server/src/main/java/tn/mnlr/vripper/{services/domain => }/tasks/LeaveThanksRunnable.java (99%)
rename vripper-server/src/main/java/tn/mnlr/vripper/{services/domain => }/tasks/LinkScanRunnable.java (98%)
rename vripper-server/src/main/java/tn/mnlr/vripper/{services/domain => }/tasks/MetadataRunnable.java (98%)
diff --git a/vripper-server/pom.xml b/vripper-server/pom.xml
index 4a68b3f..56b7e30 100644
--- a/vripper-server/pom.xml
+++ b/vripper-server/pom.xml
@@ -59,7 +59,7 @@
net.sourceforge.htmlcleaner
htmlcleaner
- 2.22
+ 2.24
io.projectreactor
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJobWrapper.java
similarity index 95%
rename from vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java
rename to vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJobWrapper.java
index 79521c7..89e03cc 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJobWrapper.java
@@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.DataService;
import java.time.LocalDateTime;
@Slf4j
-public class DownloadRunnable implements Runnable {
+public class DownloadJobWrapper implements Runnable {
private final DownloadService downloadService;
private final DataService dataService;
@@ -22,7 +22,7 @@ public class DownloadRunnable implements Runnable {
private final DownloadJob downloadJob;
- public DownloadRunnable(final DownloadJob downloadJob) {
+ public DownloadJobWrapper(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
connectionService = SpringContext.getBean(ConnectionService.class);
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java
index 19fcca4..62b0953 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadService.java
@@ -11,10 +11,14 @@ import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.SettingsService;
+import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
import java.util.*;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@@ -27,41 +31,38 @@ public class DownloadService {
private final ConcurrentHashMap threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
- private final BlockingQueue executionQueue = new LinkedBlockingQueue<>();
- private final List executing = Collections.synchronizedList(new ArrayList<>());
+ private final List running = Collections.synchronizedList(new ArrayList<>());
+ private final List pending = new ArrayList<>();
- private final PendingQueue pendingQueue;
- private final SettingsService settings;
+ private final SettingsService settingsService;
private final DataService dataService;
private final PostService postService;
+ private final List hosts;
+
private boolean pauseQ = false;
- private Thread executionThread;
private Thread pollThread;
@Autowired
public DownloadService(
- PendingQueue pendingQueue,
- SettingsService settings,
+ SettingsService settingsService,
DataService dataService,
- PostService postService) {
- this.pendingQueue = pendingQueue;
- this.settings = settings;
+ PostService postService,
+ List hosts) {
+ this.settingsService = settingsService;
this.dataService = dataService;
this.postService = postService;
+ this.hosts = hosts;
}
@PostConstruct
private void init() {
- executionThread = new Thread(this::start, "Executor thread");
- pollThread = new Thread(this::poll, "Polling thread");
+ pollThread = new Thread(this::start, "Polling thread");
pollThread.start();
- executionThread.start();
}
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
- executionThread.interrupt();
pollThread.interrupt();
executor.shutdown();
dataService
@@ -76,7 +77,7 @@ public class DownloadService {
private void stopRunning(@NonNull String postId) {
List stopping = new ArrayList<>();
- Iterator iterator = executing.iterator();
+ Iterator iterator = running.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
@@ -125,18 +126,15 @@ public class DownloadService {
post.setStatus(Status.PENDING);
dataService.updatePostStatus(post.getStatus(), post.getId());
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
- for (Image image : images) {
- try {
- pendingQueue.put(post, image);
- } catch (InterruptedException e) {
- log.warn("Thread was interrupted", e);
- Thread.currentThread().interrupt();
- }
- }
+ enqueue(post, images);
}
private boolean isPending(String postId) {
- return pendingQueue.isPending(postId);
+ return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
+ }
+
+ private boolean isRunning(String postId) {
+ return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private void stop(String postId) {
@@ -149,7 +147,7 @@ public class DownloadService {
if (FINISHED.contains(post.getStatus())) {
return;
}
- pendingQueue.stop(post);
+ pending.removeIf(p -> p.getPost().equals(post));
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
@@ -161,16 +159,12 @@ public class DownloadService {
private boolean canRun(Host host) {
boolean canRun;
- AtomicInteger count = threadCount.get(host);
- if (count == null) {
- threadCount.put(host, new AtomicInteger(0));
- }
+ int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
canRun =
- threadCount.get(host).get() < settings.getSettings().getMaxThreads()
- && (settings.getSettings().getMaxTotalThreads() == 0
- ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE
- : threadCount.values().stream().mapToInt(AtomicInteger::get).sum()
- < settings.getSettings().getMaxTotalThreads());
+ threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
+ && (settingsService.getSettings().getMaxTotalThreads() == 0
+ ? totalRunning < MAX_POOL_SIZE
+ : totalRunning < settingsService.getSettings().getMaxTotalThreads());
if (canRun && !pauseQ) {
threadCount.get(host).incrementAndGet();
return true;
@@ -178,59 +172,102 @@ public class DownloadService {
return false;
}
- private void poll() {
- while (!Thread.interrupted()) {
- try {
- List peek = pendingQueue.peek();
- for (DownloadJob downloadJob : peek) {
- if (canRun(downloadJob.getImage().getHost())) {
- executionQueue.offer(downloadJob);
- pendingQueue.remove(downloadJob);
+ private Map candidateCount() {
+ HashMap map = new HashMap<>();
+ hosts.forEach(
+ h -> {
+ AtomicInteger count = threadCount.get(h);
+ if (count == null) {
+ count = new AtomicInteger(0);
+ threadCount.put(h, count);
}
- }
- synchronized (threadCount) {
- threadCount.wait(2_000);
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
+ });
+ return map;
+ }
+
+ private List getCandidates(Map candidateCount, int max) {
+ int i = 0;
+ List candidates = new ArrayList<>();
+ for (DownloadJob downloadJob : pending) {
+ Host host = downloadJob.getImage().getHost();
+ Integer maxPerHost = candidateCount.get(host);
+ if (maxPerHost > 0 && i < max) {
+ candidates.add(downloadJob);
+ candidateCount.put(host, maxPerHost + 1);
+ i++;
+ }
+ if (i >= max) {
break;
}
}
+ return candidates;
+ }
+
+ public void enqueue(Post post, Collection imageList) {
+ synchronized (this) {
+ for (Image image : imageList) {
+ log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
+ image.init();
+ dataService.updateImageStatus(image.getStatus(), image.getId());
+ dataService.updateImageCurrent(image.getCurrent(), image.getId());
+ DownloadJob downloadJob =
+ new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
+ pending.add(downloadJob);
+ }
+ pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
+
+ this.notify();
+ }
}
private void start() {
while (!Thread.interrupted()) {
try {
- push(executionQueue.take());
+ synchronized (this) {
+ List accepted = new ArrayList<>();
+ List candidates = getCandidates(candidateCount(), MAX_POOL_SIZE);
+ candidates.forEach(
+ c -> {
+ if (canRun(c.getImage().getHost())) {
+ accepted.add(c);
+ }
+ });
+ pending.removeAll(accepted);
+ accepted.forEach(this::push);
+ accepted.clear();
+ this.wait();
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
- } catch (Exception e) {
- log.error("Execution Service failed", e);
- break;
}
}
}
private void push(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
- executor.execute(new DownloadRunnable(downloadJob));
- executing.add(downloadJob);
+ executor.execute(new DownloadJobWrapper(downloadJob));
+ running.add(downloadJob);
}
- public synchronized void afterJobFinish(DownloadJob downloadJob) {
- int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
- if (count == 0) {
- dataService.finishPost(downloadJob.getPost());
- }
- threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
- executing.remove(downloadJob);
- synchronized (threadCount) {
- threadCount.notify();
+ public void afterJobFinish(DownloadJob downloadJob) {
+ synchronized (this) {
+ running.remove(downloadJob);
+ threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
+ if (!isPending(downloadJob.getPost().getPostId())
+ && !isRunning(downloadJob.getPost().getPostId())) {
+ dataService.finishPost(downloadJob.getPost());
+ }
+ this.notify();
}
}
+ public int pendingCount() {
+ return pending.size();
+ }
+
public int runningCount() {
- return executing.size();
+ return running.size();
}
}
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java
deleted file mode 100644
index 8d74acd..0000000
--- a/vripper-server/src/main/java/tn/mnlr/vripper/download/PendingQueue.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package tn.mnlr.vripper.download;
-
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-import tn.mnlr.vripper.host.Host;
-import tn.mnlr.vripper.jpa.domain.Image;
-import tn.mnlr.vripper.jpa.domain.Post;
-import tn.mnlr.vripper.services.DataService;
-import tn.mnlr.vripper.services.SettingsService;
-import tn.mnlr.vripper.services.domain.Settings;
-
-import javax.annotation.PostConstruct;
-import java.util.*;
-import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-
-@Service
-@Slf4j
-public class PendingQueue {
-
- private final DataService dataService;
- private final SettingsService settingsService;
- private final List hosts;
-
- private final ConcurrentHashMap> pendingQ =
- new ConcurrentHashMap<>();
- private final ConcurrentHashMap toBeExecuted = new ConcurrentHashMap<>();
-
- @Autowired
- public PendingQueue(DataService dataService, SettingsService settingsService, List hosts) {
- this.dataService = dataService;
- this.settingsService = settingsService;
- this.hosts = hosts;
- }
-
- @PostConstruct
- private void init() {
- hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
- }
-
- public void put(Post post, Image image) throws InterruptedException {
- log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
- image.init();
- dataService.updateImageStatus(image.getStatus(), image.getId());
- dataService.updateImageCurrent(image.getCurrent(), image.getId());
- DownloadJob downloadJob =
- new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
- pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
- checkKey(post.getPostId());
- toBeExecuted.get(post.getPostId()).incrementAndGet();
- }
-
- private synchronized void checkKey(String postId) {
- if (!toBeExecuted.containsKey(postId)) {
- toBeExecuted.put(postId, new AtomicInteger(0));
- }
- }
-
- public void remove(final DownloadJob downloadJob) {
- pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
- }
-
- public List peek() {
- List downloadJobs = new ArrayList<>();
- if (hosts.size() == 0) {
- return downloadJobs;
- }
- for (Host host : hosts) {
- Iterator it = pendingQ.get(host).iterator();
- for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
- DownloadJob downloadJob = it.hasNext() ? it.next() : null;
- if (downloadJob != null) {
- downloadJobs.add(downloadJob);
- }
- }
- }
- return downloadJobs;
- }
-
- public void enqueue(Post post, Set images) throws InterruptedException {
- for (Image image : images) {
- put(post, image);
- }
- }
-
- public int size() {
- return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
- }
-
- public void stop(Post post) {
- Predicate predicate = next -> next.getImage().getPostId().equals(post.getPostId());
- for (Map.Entry> entry : pendingQ.entrySet()) {
- entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
- entry.getValue().removeIf(predicate);
- decrement(post.getPostId());
- }
- }
-
- public boolean isPending(String postId) {
- return toBeExecuted.containsKey(postId);
- }
-
- public synchronized int decrement(String postId) {
- AtomicInteger counter = toBeExecuted.get(postId);
- if (counter == null) {
- return 0;
- }
- int count = counter.decrementAndGet();
- if (count == 0) {
- toBeExecuted.remove(postId);
- }
- return count;
- }
-}
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java b/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java
index 494cd9f..985324d 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/event/Event.java
@@ -28,6 +28,7 @@ public class Event {
LOG_EVENT_REMOVE,
VG_USER,
GLOBAL_STATE,
- BYTES_PER_SECOND
+ BYTES_PER_SECOND,
+ SETTINGS_UPDATE
}
}
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java
index 1dd4be4..4f26b0c 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java
@@ -7,6 +7,7 @@ import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.jpa.domain.enums.Status;
+import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@@ -49,6 +50,8 @@ public class Post {
private Metadata metadata;
+ private LocalDateTime addedOn;
+
public Post(
String title,
String url,
@@ -65,6 +68,7 @@ public class Post {
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
+ addedOn = LocalDateTime.now();
}
@Override
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java
index 821fe04..7269c8e 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java
@@ -13,6 +13,7 @@ import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Timestamp;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -37,7 +38,7 @@ public class PostRepository implements IPostRepository {
public Post save(Post post) {
long id = nextId();
jdbcTemplate.update(
- "INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+ "INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
@@ -52,7 +53,8 @@ public class PostRepository implements IPostRepository {
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
- post.getUrl());
+ post.getUrl(),
+ Timestamp.valueOf(post.getAddedOn()));
post.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return post;
@@ -198,6 +200,7 @@ class PostRowMapper implements RowMapper {
if ((previews = rs.getString("post.PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
+ post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
Long metadataId = rs.getLong("metadata.POST_ID_REF");
if (!rs.wasNull()) {
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java b/vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java
deleted file mode 100644
index c3e1bb0..0000000
--- a/vripper-server/src/main/java/tn/mnlr/vripper/listener/DataEventListener.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package tn.mnlr.vripper.listener;
-
-import reactor.core.publisher.Flux;
-
-public interface DataEventListener {
- Flux getDataFlux();
-}
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java b/vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java
deleted file mode 100644
index b17b6ed..0000000
--- a/vripper-server/src/main/java/tn/mnlr/vripper/listener/EmitHandler.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package tn.mnlr.vripper.listener;
-
-import reactor.core.publisher.Sinks.EmitFailureHandler;
-
-public class EmitHandler {
- public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
-}
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java
index 630687a..c4f9952 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionService.java
@@ -17,8 +17,9 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
-import reactor.core.publisher.Flux;
import tn.mnlr.vripper.download.DownloadJob;
+import tn.mnlr.vripper.event.Event;
+import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PreDestroy;
@@ -41,22 +42,25 @@ public class ConnectionService {
private int connectionTimeout;
private int maxAttempts;
- public ConnectionService(SettingsService settingsService) {
+ public ConnectionService(EventBus eventBus, SettingsService settingsService) {
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
maxAttempts = settingsService.getSettings().getMaxAttempts();
- Flux settingsFlux = settingsService.getSettingsFlux();
disposable =
- settingsFlux.subscribe(
- settings -> {
- if (connectionTimeout != settings.getConnectionTimeout()) {
- connectionTimeout = settings.getConnectionTimeout();
- buildRequestConfig();
- }
- if (maxAttempts != settings.getMaxAttempts()) {
- maxAttempts = settings.getMaxAttempts();
- buildRetryPolicy();
- }
- });
+ eventBus
+ .flux()
+ .filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
+ .map(e -> ((Settings) e.getData()))
+ .subscribe(
+ settings -> {
+ if (connectionTimeout != settings.getConnectionTimeout()) {
+ connectionTimeout = settings.getConnectionTimeout();
+ buildRequestConfig();
+ }
+ if (maxAttempts != settings.getMaxAttempts()) {
+ maxAttempts = settings.getMaxAttempts();
+ buildRetryPolicy();
+ }
+ });
buildRequestConfig();
buildRetryPolicy();
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java
index 95332df..73aef1e 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java
@@ -6,7 +6,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.download.DownloadService;
-import tn.mnlr.vripper.download.PendingQueue;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.services.domain.GlobalState;
@@ -15,7 +14,6 @@ import tn.mnlr.vripper.services.domain.GlobalState;
@EnableScheduling
public class GlobalStateService {
- private final PendingQueue pendingQueue;
private final DownloadService downloadService;
private final DataService dataService;
private final EventBus eventBus;
@@ -23,11 +21,7 @@ public class GlobalStateService {
@Autowired
public GlobalStateService(
- PendingQueue pendingQueue,
- DownloadService downloadService,
- DataService dataService,
- EventBus eventBus) {
- this.pendingQueue = pendingQueue;
+ DownloadService downloadService, DataService dataService, EventBus eventBus) {
this.downloadService = downloadService;
this.dataService = dataService;
this.eventBus = eventBus;
@@ -37,7 +31,9 @@ public class GlobalStateService {
private void interval() {
GlobalState newGlobalState =
new GlobalState(
- downloadService.runningCount(), pendingQueue.size(), dataService.countErrorImages());
+ downloadService.runningCount(),
+ downloadService.pendingCount(),
+ dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java
index fa3219a..18533cf 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java
@@ -12,12 +12,16 @@ import org.apache.http.util.EntityUtils;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
+import reactor.core.Disposable;
+import tn.mnlr.vripper.event.Event;
+import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
-import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable;
+import tn.mnlr.vripper.tasks.MetadataRunnable;
+import javax.annotation.PreDestroy;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -38,19 +42,33 @@ public class MetadataService {
private final XpathService xpathService;
private final Map fetchingMetadata = new ConcurrentHashMap<>();
private final ThreadPoolService threadPoolService;
+ private final Disposable disposable;
public MetadataService(
ConnectionService cm,
VGAuthService VGAuthService,
HtmlProcessorService htmlProcessorService,
XpathService xpathService,
- ThreadPoolService threadPoolService) {
+ ThreadPoolService threadPoolService,
+ EventBus eventBus) {
this.cm = cm;
this.VGAuthService = VGAuthService;
this.htmlProcessorService = htmlProcessorService;
this.xpathService = xpathService;
this.threadPoolService = threadPoolService;
cache = Caffeine.newBuilder().expireAfterWrite(30, TimeUnit.MINUTES).build(this::fetchMetadata);
+ disposable =
+ eventBus
+ .flux()
+ .filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
+ .subscribe(e -> this.cache.invalidateAll());
+ }
+
+ @PreDestroy
+ private void destroy() {
+ if (disposable != null) {
+ disposable.dispose();
+ }
}
public Metadata get(Post post) throws ExecutionException {
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java
index 1b26aa8..ce00c34 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java
@@ -5,13 +5,17 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import reactor.core.Disposable;
+import tn.mnlr.vripper.event.Event;
+import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.domain.MultiPostScanParser;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
-import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
-import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable;
+import tn.mnlr.vripper.tasks.AddPostRunnable;
+import tn.mnlr.vripper.tasks.AddQueuedRunnable;
+import javax.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -24,12 +28,14 @@ public class PostService {
private final ThreadPoolService threadPoolService;
private final MetadataService metadataService;
private final LoadingCache cache;
+ private final Disposable disposable;
@Autowired
public PostService(
DataService dataService,
ThreadPoolService threadPoolService,
- MetadataService metadataService) {
+ MetadataService metadataService,
+ EventBus eventBus) {
this.dataService = dataService;
this.threadPoolService = threadPoolService;
this.metadataService = metadataService;
@@ -37,6 +43,18 @@ public class PostService {
Caffeine.newBuilder()
.expireAfterWrite(5, TimeUnit.MINUTES)
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
+ disposable =
+ eventBus
+ .flux()
+ .filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
+ .subscribe(e -> this.cache.invalidateAll());
+ }
+
+ @PreDestroy
+ private void destroy() {
+ if (disposable != null) {
+ disposable.dispose();
+ }
}
public void stopFetchingMetadata(Post post) {
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java
index bcc5256..25a3c7e 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java
@@ -12,11 +12,10 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
-import reactor.core.publisher.Flux;
-import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.event.Event;
+import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.ValidationException;
-import tn.mnlr.vripper.listener.EmitHandler;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
@@ -46,7 +45,7 @@ public class SettingsService {
private final Set proxies = new HashSet<>();
- private Sinks.Many sink = Sinks.many().multicast().onBackpressureBuffer();
+ private final EventBus eventBus;
@Getter private Settings settings = new Settings();
@@ -54,7 +53,10 @@ public class SettingsService {
private Resource defaultProxies;
public SettingsService(
- @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
+ @Value("${base.dir}") String baseDir,
+ @Value("${base.dir.name}") String baseDirName,
+ EventBus eventBus) {
+ this.eventBus = eventBus;
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -64,7 +66,7 @@ public class SettingsService {
private void init() {
loadViperProxies();
restore();
- sink.emitNext(settings, EmitHandler.RETRY);
+ eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
private void loadViperProxies() {
@@ -93,10 +95,6 @@ public class SettingsService {
proxies.add("https://vipergirls.to");
}
- Flux getSettingsFlux() {
- return sink.asFlux();
- }
-
public List getProxies() {
return new ArrayList<>(proxies);
}
@@ -116,7 +114,7 @@ public class SettingsService {
this.settings = settings;
save();
- sink.emitNext(settings, EmitHandler.RETRY);
+ eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
public void restore() {
@@ -232,7 +230,6 @@ public class SettingsService {
@PreDestroy
private void destroy() {
save();
- sink.emitComplete(EmitHandler.RETRY);
}
public void check(Settings settings) throws ValidationException {
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java
index de5523b..b6ddc67 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java
@@ -19,7 +19,7 @@ import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.jpa.domain.Post;
-import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable;
+import tn.mnlr.vripper.tasks.LeaveThanksRunnable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@@ -51,8 +51,12 @@ public class VGAuthService {
this.cm = cm;
this.settingsService = settingsService;
this.threadPoolService = threadPoolService;
- disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate());
this.eventBus = eventBus;
+ disposable =
+ eventBus
+ .flux()
+ .filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE))
+ .subscribe(e -> authenticate());
}
@PostConstruct
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java
index ba16a60..9cd1ba6 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MultiPostScanParser.java
@@ -68,8 +68,6 @@ public class MultiPostScanParser {
}
try {
- //
- // System.out.println(EntityUtils.toString(response.getEntity()));
factory
.newSAXParser()
.parse(
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddPostRunnable.java
similarity index 92%
rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java
rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddPostRunnable.java
index daacf2e..2d32649 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddPostRunnable.java
@@ -1,9 +1,9 @@
-package tn.mnlr.vripper.services.domain.tasks;
+package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
-import tn.mnlr.vripper.download.PendingQueue;
+import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.LogEvent;
@@ -31,11 +31,11 @@ public class AddPostRunnable implements Runnable {
private final DataService dataService;
private final MetadataService metadataService;
private final SettingsService settingsService;
- private final PendingQueue pendingQueue;
private final VGAuthService VGAuthService;
private final LogEvent logEvent;
private final ILogEventRepository eventRepository;
private final String link;
+ private final DownloadService downloadService;
public AddPostRunnable(String postId, String threadId) {
this.postId = postId;
@@ -43,7 +43,7 @@ public class AddPostRunnable implements Runnable {
this.dataService = SpringContext.getBean(DataService.class);
this.metadataService = SpringContext.getBean(MetadataService.class);
this.settingsService = SpringContext.getBean(SettingsService.class);
- this.pendingQueue = SpringContext.getBean(PendingQueue.class);
+ this.downloadService = SpringContext.getBean(DownloadService.class);
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
this.eventRepository = SpringContext.getBean(ILogEventRepository.class);
link =
@@ -112,13 +112,7 @@ public class AddPostRunnable implements Runnable {
if (settingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
- try {
- pendingQueue.enqueue(post, images);
- } catch (InterruptedException e) {
- log.warn("Interruption was caught");
- Thread.currentThread().interrupt();
- return;
- }
+ downloadService.enqueue(post, images);
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddQueuedRunnable.java
similarity index 98%
rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java
rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddQueuedRunnable.java
index 9d95841..daff6e5 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/AddQueuedRunnable.java
@@ -1,4 +1,4 @@
-package tn.mnlr.vripper.services.domain.tasks;
+package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import tn.mnlr.vripper.SpringContext;
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LeaveThanksRunnable.java
similarity index 99%
rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java
rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/LeaveThanksRunnable.java
index 489a329..9e8515d 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LeaveThanksRunnable.java
@@ -1,4 +1,4 @@
-package tn.mnlr.vripper.services.domain.tasks;
+package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LinkScanRunnable.java
similarity index 98%
rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java
rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/LinkScanRunnable.java
index acbba58..8b14f31 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/LinkScanRunnable.java
@@ -1,4 +1,4 @@
-package tn.mnlr.vripper.services.domain.tasks;
+package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import org.springframework.lang.NonNull;
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/MetadataRunnable.java
similarity index 98%
rename from vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java
rename to vripper-server/src/main/java/tn/mnlr/vripper/tasks/MetadataRunnable.java
index aa68195..cc578c8 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/tasks/MetadataRunnable.java
@@ -1,4 +1,4 @@
-package tn.mnlr.vripper.services.domain.tasks;
+package tn.mnlr.vripper.tasks;
import lombok.Getter;
import lombok.NonNull;
diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java
index 16580f3..ad14ea2 100644
--- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java
+++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java
@@ -12,8 +12,8 @@ import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.domain.MultiPostItem;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
-import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
-import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable;
+import tn.mnlr.vripper.tasks.AddPostRunnable;
+import tn.mnlr.vripper.tasks.LinkScanRunnable;
import tn.mnlr.vripper.web.restendpoints.domain.*;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
diff --git a/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml b/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml
index a633bac..63cdb32 100644
--- a/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml
+++ b/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml
@@ -161,4 +161,11 @@
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
referencedColumnNames="ID" referencedTableName="POST"/>
+
+
+
+
+
+
+