mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Enhance download queue logic
This commit is contained in:
@@ -59,7 +59,7 @@
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.htmlcleaner</groupId>
|
||||
<artifactId>htmlcleaner</artifactId>
|
||||
<version>2.22</version>
|
||||
<version>2.24</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
|
||||
+2
-2
@@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.DataService;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Slf4j
|
||||
public class DownloadRunnable implements Runnable {
|
||||
public class DownloadJobWrapper implements Runnable {
|
||||
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
@@ -22,7 +22,7 @@ public class DownloadRunnable implements Runnable {
|
||||
|
||||
private final DownloadJob downloadJob;
|
||||
|
||||
public DownloadRunnable(final DownloadJob downloadJob) {
|
||||
public DownloadJobWrapper(final DownloadJob downloadJob) {
|
||||
downloadService = SpringContext.getBean(DownloadService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
connectionService = SpringContext.getBean(ConnectionService.class);
|
||||
@@ -11,10 +11,14 @@ import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Service
|
||||
@@ -27,41 +31,38 @@ public class DownloadService {
|
||||
|
||||
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
|
||||
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
|
||||
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
|
||||
private final List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
|
||||
private final List<DownloadJob> pending = new ArrayList<>();
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final SettingsService settings;
|
||||
private final SettingsService settingsService;
|
||||
private final DataService dataService;
|
||||
private final PostService postService;
|
||||
|
||||
private final List<Host> hosts;
|
||||
|
||||
private boolean pauseQ = false;
|
||||
private Thread executionThread;
|
||||
private Thread pollThread;
|
||||
|
||||
@Autowired
|
||||
public DownloadService(
|
||||
PendingQueue pendingQueue,
|
||||
SettingsService settings,
|
||||
SettingsService settingsService,
|
||||
DataService dataService,
|
||||
PostService postService) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
this.settings = settings;
|
||||
PostService postService,
|
||||
List<Host> hosts) {
|
||||
this.settingsService = settingsService;
|
||||
this.dataService = dataService;
|
||||
this.postService = postService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
executionThread = new Thread(this::start, "Executor thread");
|
||||
pollThread = new Thread(this::poll, "Polling thread");
|
||||
pollThread = new Thread(this::start, "Polling thread");
|
||||
pollThread.start();
|
||||
executionThread.start();
|
||||
}
|
||||
|
||||
public void destroy() throws Exception {
|
||||
log.info("Shutting down ExecutionService");
|
||||
executionThread.interrupt();
|
||||
pollThread.interrupt();
|
||||
executor.shutdown();
|
||||
dataService
|
||||
@@ -76,7 +77,7 @@ public class DownloadService {
|
||||
|
||||
private void stopRunning(@NonNull String postId) {
|
||||
List<DownloadJob> stopping = new ArrayList<>();
|
||||
Iterator<DownloadJob> iterator = executing.iterator();
|
||||
Iterator<DownloadJob> iterator = running.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob downloadJob = iterator.next();
|
||||
if (postId.equals(downloadJob.getPost().getPostId())) {
|
||||
@@ -125,18 +126,15 @@ public class DownloadService {
|
||||
post.setStatus(Status.PENDING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
for (Image image : images) {
|
||||
try {
|
||||
pendingQueue.put(post, image);
|
||||
} catch (InterruptedException e) {
|
||||
log.warn("Thread was interrupted", e);
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
enqueue(post, images);
|
||||
}
|
||||
|
||||
private boolean isPending(String postId) {
|
||||
return pendingQueue.isPending(postId);
|
||||
return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
|
||||
}
|
||||
|
||||
private boolean isRunning(String postId) {
|
||||
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
|
||||
}
|
||||
|
||||
private void stop(String postId) {
|
||||
@@ -149,7 +147,7 @@ public class DownloadService {
|
||||
if (FINISHED.contains(post.getStatus())) {
|
||||
return;
|
||||
}
|
||||
pendingQueue.stop(post);
|
||||
pending.removeIf(p -> p.getPost().equals(post));
|
||||
stopRunning(postId);
|
||||
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
|
||||
dataService.finishPost(post);
|
||||
@@ -161,16 +159,12 @@ public class DownloadService {
|
||||
|
||||
private boolean canRun(Host host) {
|
||||
boolean canRun;
|
||||
AtomicInteger count = threadCount.get(host);
|
||||
if (count == null) {
|
||||
threadCount.put(host, new AtomicInteger(0));
|
||||
}
|
||||
int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
canRun =
|
||||
threadCount.get(host).get() < settings.getSettings().getMaxThreads()
|
||||
&& (settings.getSettings().getMaxTotalThreads() == 0
|
||||
? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE
|
||||
: threadCount.values().stream().mapToInt(AtomicInteger::get).sum()
|
||||
< settings.getSettings().getMaxTotalThreads());
|
||||
threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
|
||||
&& (settingsService.getSettings().getMaxTotalThreads() == 0
|
||||
? totalRunning < MAX_POOL_SIZE
|
||||
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
|
||||
if (canRun && !pauseQ) {
|
||||
threadCount.get(host).incrementAndGet();
|
||||
return true;
|
||||
@@ -178,59 +172,102 @@ public class DownloadService {
|
||||
return false;
|
||||
}
|
||||
|
||||
private void poll() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
List<DownloadJob> peek = pendingQueue.peek();
|
||||
for (DownloadJob downloadJob : peek) {
|
||||
if (canRun(downloadJob.getImage().getHost())) {
|
||||
executionQueue.offer(downloadJob);
|
||||
pendingQueue.remove(downloadJob);
|
||||
private Map<Host, Integer> candidateCount() {
|
||||
HashMap<Host, Integer> map = new HashMap<>();
|
||||
hosts.forEach(
|
||||
h -> {
|
||||
AtomicInteger count = threadCount.get(h);
|
||||
if (count == null) {
|
||||
count = new AtomicInteger(0);
|
||||
threadCount.put(h, count);
|
||||
}
|
||||
}
|
||||
synchronized (threadCount) {
|
||||
threadCount.wait(2_000);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount, int max) {
|
||||
int i = 0;
|
||||
List<DownloadJob> candidates = new ArrayList<>();
|
||||
for (DownloadJob downloadJob : pending) {
|
||||
Host host = downloadJob.getImage().getHost();
|
||||
Integer maxPerHost = candidateCount.get(host);
|
||||
if (maxPerHost > 0 && i < max) {
|
||||
candidates.add(downloadJob);
|
||||
candidateCount.put(host, maxPerHost + 1);
|
||||
i++;
|
||||
}
|
||||
if (i >= max) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
public void enqueue(Post post, Collection<Image> imageList) {
|
||||
synchronized (this) {
|
||||
for (Image image : imageList) {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob =
|
||||
new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
|
||||
pending.add(downloadJob);
|
||||
}
|
||||
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
|
||||
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
private void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
push(executionQueue.take());
|
||||
synchronized (this) {
|
||||
List<DownloadJob> accepted = new ArrayList<>();
|
||||
List<DownloadJob> candidates = getCandidates(candidateCount(), MAX_POOL_SIZE);
|
||||
candidates.forEach(
|
||||
c -> {
|
||||
if (canRun(c.getImage().getHost())) {
|
||||
accepted.add(c);
|
||||
}
|
||||
});
|
||||
pending.removeAll(accepted);
|
||||
accepted.forEach(this::push);
|
||||
accepted.clear();
|
||||
this.wait();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
log.error("Execution Service failed", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void push(DownloadJob downloadJob) {
|
||||
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
|
||||
executor.execute(new DownloadRunnable(downloadJob));
|
||||
executing.add(downloadJob);
|
||||
executor.execute(new DownloadJobWrapper(downloadJob));
|
||||
running.add(downloadJob);
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(DownloadJob downloadJob) {
|
||||
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
|
||||
if (count == 0) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
executing.remove(downloadJob);
|
||||
synchronized (threadCount) {
|
||||
threadCount.notify();
|
||||
public void afterJobFinish(DownloadJob downloadJob) {
|
||||
synchronized (this) {
|
||||
running.remove(downloadJob);
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
if (!isPending(downloadJob.getPost().getPostId())
|
||||
&& !isRunning(downloadJob.getPost().getPostId())) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
public int pendingCount() {
|
||||
return pending.size();
|
||||
}
|
||||
|
||||
public int runningCount() {
|
||||
return executing.size();
|
||||
return running.size();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.BlockingDeque;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PendingQueue {
|
||||
|
||||
private final DataService dataService;
|
||||
private final SettingsService settingsService;
|
||||
private final List<Host> hosts;
|
||||
|
||||
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ =
|
||||
new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
|
||||
|
||||
@Autowired
|
||||
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
|
||||
this.dataService = dataService;
|
||||
this.settingsService = settingsService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
|
||||
}
|
||||
|
||||
public void put(Post post, Image image) throws InterruptedException {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob =
|
||||
new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
|
||||
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
|
||||
checkKey(post.getPostId());
|
||||
toBeExecuted.get(post.getPostId()).incrementAndGet();
|
||||
}
|
||||
|
||||
private synchronized void checkKey(String postId) {
|
||||
if (!toBeExecuted.containsKey(postId)) {
|
||||
toBeExecuted.put(postId, new AtomicInteger(0));
|
||||
}
|
||||
}
|
||||
|
||||
public void remove(final DownloadJob downloadJob) {
|
||||
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
|
||||
}
|
||||
|
||||
public List<DownloadJob> peek() {
|
||||
List<DownloadJob> downloadJobs = new ArrayList<>();
|
||||
if (hosts.size() == 0) {
|
||||
return downloadJobs;
|
||||
}
|
||||
for (Host host : hosts) {
|
||||
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
|
||||
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
|
||||
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
|
||||
if (downloadJob != null) {
|
||||
downloadJobs.add(downloadJob);
|
||||
}
|
||||
}
|
||||
}
|
||||
return downloadJobs;
|
||||
}
|
||||
|
||||
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
|
||||
for (Image image : images) {
|
||||
put(post, image);
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
}
|
||||
|
||||
public void stop(Post post) {
|
||||
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
|
||||
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
|
||||
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
|
||||
entry.getValue().removeIf(predicate);
|
||||
decrement(post.getPostId());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isPending(String postId) {
|
||||
return toBeExecuted.containsKey(postId);
|
||||
}
|
||||
|
||||
public synchronized int decrement(String postId) {
|
||||
AtomicInteger counter = toBeExecuted.get(postId);
|
||||
if (counter == null) {
|
||||
return 0;
|
||||
}
|
||||
int count = counter.decrementAndGet();
|
||||
if (count == 0) {
|
||||
toBeExecuted.remove(postId);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ public class Event<T> {
|
||||
LOG_EVENT_REMOVE,
|
||||
VG_USER,
|
||||
GLOBAL_STATE,
|
||||
BYTES_PER_SECOND
|
||||
BYTES_PER_SECOND,
|
||||
SETTINGS_UPDATE
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
@@ -49,6 +50,8 @@ public class Post {
|
||||
|
||||
private Metadata metadata;
|
||||
|
||||
private LocalDateTime addedOn;
|
||||
|
||||
public Post(
|
||||
String title,
|
||||
String url,
|
||||
@@ -65,6 +68,7 @@ public class Post {
|
||||
this.threadTitle = threadTitle;
|
||||
this.securityToken = securityToken;
|
||||
status = Status.STOPPED;
|
||||
addedOn = LocalDateTime.now();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+5
-2
@@ -13,6 +13,7 @@ import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
@@ -37,7 +38,7 @@ public class PostRepository implements IPostRepository {
|
||||
public Post save(Post post) {
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
post.getDone(),
|
||||
post.getForum(),
|
||||
@@ -52,7 +53,8 @@ public class PostRepository implements IPostRepository {
|
||||
post.getThreadTitle(),
|
||||
post.getTitle(),
|
||||
post.getTotal(),
|
||||
post.getUrl());
|
||||
post.getUrl(),
|
||||
Timestamp.valueOf(post.getAddedOn()));
|
||||
post.setId(id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return post;
|
||||
@@ -198,6 +200,7 @@ class PostRowMapper implements RowMapper<Post> {
|
||||
if ((previews = rs.getString("post.PREVIEWS")) != null) {
|
||||
post.setPreviews(Set.of(previews.split(DELIMITER)));
|
||||
}
|
||||
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
|
||||
|
||||
Long metadataId = rs.getLong("metadata.POST_ID_REF");
|
||||
if (!rs.wasNull()) {
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public interface DataEventListener<T> {
|
||||
Flux<T> getDataFlux();
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Sinks.EmitFailureHandler;
|
||||
|
||||
public class EmitHandler {
|
||||
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
|
||||
}
|
||||
@@ -17,8 +17,9 @@ import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.Disposable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import tn.mnlr.vripper.download.DownloadJob;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
@@ -41,22 +42,25 @@ public class ConnectionService {
|
||||
private int connectionTimeout;
|
||||
private int maxAttempts;
|
||||
|
||||
public ConnectionService(SettingsService settingsService) {
|
||||
public ConnectionService(EventBus eventBus, SettingsService settingsService) {
|
||||
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
|
||||
maxAttempts = settingsService.getSettings().getMaxAttempts();
|
||||
Flux<Settings> settingsFlux = settingsService.getSettingsFlux();
|
||||
disposable =
|
||||
settingsFlux.subscribe(
|
||||
settings -> {
|
||||
if (connectionTimeout != settings.getConnectionTimeout()) {
|
||||
connectionTimeout = settings.getConnectionTimeout();
|
||||
buildRequestConfig();
|
||||
}
|
||||
if (maxAttempts != settings.getMaxAttempts()) {
|
||||
maxAttempts = settings.getMaxAttempts();
|
||||
buildRetryPolicy();
|
||||
}
|
||||
});
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.map(e -> ((Settings) e.getData()))
|
||||
.subscribe(
|
||||
settings -> {
|
||||
if (connectionTimeout != settings.getConnectionTimeout()) {
|
||||
connectionTimeout = settings.getConnectionTimeout();
|
||||
buildRequestConfig();
|
||||
}
|
||||
if (maxAttempts != settings.getMaxAttempts()) {
|
||||
maxAttempts = settings.getMaxAttempts();
|
||||
buildRetryPolicy();
|
||||
}
|
||||
});
|
||||
|
||||
buildRequestConfig();
|
||||
buildRetryPolicy();
|
||||
|
||||
@@ -6,7 +6,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.download.PendingQueue;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.services.domain.GlobalState;
|
||||
@@ -15,7 +14,6 @@ import tn.mnlr.vripper.services.domain.GlobalState;
|
||||
@EnableScheduling
|
||||
public class GlobalStateService {
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final EventBus eventBus;
|
||||
@@ -23,11 +21,7 @@ public class GlobalStateService {
|
||||
|
||||
@Autowired
|
||||
public GlobalStateService(
|
||||
PendingQueue pendingQueue,
|
||||
DownloadService downloadService,
|
||||
DataService dataService,
|
||||
EventBus eventBus) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
DownloadService downloadService, DataService dataService, EventBus eventBus) {
|
||||
this.downloadService = downloadService;
|
||||
this.dataService = dataService;
|
||||
this.eventBus = eventBus;
|
||||
@@ -37,7 +31,9 @@ public class GlobalStateService {
|
||||
private void interval() {
|
||||
GlobalState newGlobalState =
|
||||
new GlobalState(
|
||||
downloadService.runningCount(), pendingQueue.size(), dataService.countErrorImages());
|
||||
downloadService.runningCount(),
|
||||
downloadService.pendingCount(),
|
||||
dataService.countErrorImages());
|
||||
if (!newGlobalState.equals(currentState)) {
|
||||
currentState = newGlobalState;
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
|
||||
|
||||
@@ -12,12 +12,16 @@ import org.apache.http.util.EntityUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import reactor.core.Disposable;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable;
|
||||
import tn.mnlr.vripper.tasks.MetadataRunnable;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
@@ -38,19 +42,33 @@ public class MetadataService {
|
||||
private final XpathService xpathService;
|
||||
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final Disposable disposable;
|
||||
|
||||
public MetadataService(
|
||||
ConnectionService cm,
|
||||
VGAuthService VGAuthService,
|
||||
HtmlProcessorService htmlProcessorService,
|
||||
XpathService xpathService,
|
||||
ThreadPoolService threadPoolService) {
|
||||
ThreadPoolService threadPoolService,
|
||||
EventBus eventBus) {
|
||||
this.cm = cm;
|
||||
this.VGAuthService = VGAuthService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
this.xpathService = xpathService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
cache = Caffeine.newBuilder().expireAfterWrite(30, TimeUnit.MINUTES).build(this::fetchMetadata);
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.subscribe(e -> this.cache.invalidateAll());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
if (disposable != null) {
|
||||
disposable.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public Metadata get(Post post) throws ExecutionException {
|
||||
|
||||
@@ -5,13 +5,17 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.Disposable;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanParser;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddQueuedRunnable;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -24,12 +28,14 @@ public class PostService {
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final MetadataService metadataService;
|
||||
private final LoadingCache<Queued, MultiPostScanResult> cache;
|
||||
private final Disposable disposable;
|
||||
|
||||
@Autowired
|
||||
public PostService(
|
||||
DataService dataService,
|
||||
ThreadPoolService threadPoolService,
|
||||
MetadataService metadataService) {
|
||||
MetadataService metadataService,
|
||||
EventBus eventBus) {
|
||||
this.dataService = dataService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.metadataService = metadataService;
|
||||
@@ -37,6 +43,18 @@ public class PostService {
|
||||
Caffeine.newBuilder()
|
||||
.expireAfterWrite(5, TimeUnit.MINUTES)
|
||||
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.subscribe(e -> this.cache.invalidateAll());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
if (disposable != null) {
|
||||
disposable.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public void stopFetchingMetadata(Post post) {
|
||||
|
||||
@@ -12,11 +12,10 @@ import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.exception.ValidationException;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
@@ -46,7 +45,7 @@ public class SettingsService {
|
||||
|
||||
private final Set<String> proxies = new HashSet<>();
|
||||
|
||||
private Sinks.Many<Settings> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Getter private Settings settings = new Settings();
|
||||
|
||||
@@ -54,7 +53,10 @@ public class SettingsService {
|
||||
private Resource defaultProxies;
|
||||
|
||||
public SettingsService(
|
||||
@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
|
||||
@Value("${base.dir}") String baseDir,
|
||||
@Value("${base.dir.name}") String baseDirName,
|
||||
EventBus eventBus) {
|
||||
this.eventBus = eventBus;
|
||||
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
|
||||
this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
|
||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
@@ -64,7 +66,7 @@ public class SettingsService {
|
||||
private void init() {
|
||||
loadViperProxies();
|
||||
restore();
|
||||
sink.emitNext(settings, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
|
||||
}
|
||||
|
||||
private void loadViperProxies() {
|
||||
@@ -93,10 +95,6 @@ public class SettingsService {
|
||||
proxies.add("https://vipergirls.to");
|
||||
}
|
||||
|
||||
Flux<Settings> getSettingsFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
public List<String> getProxies() {
|
||||
return new ArrayList<>(proxies);
|
||||
}
|
||||
@@ -116,7 +114,7 @@ public class SettingsService {
|
||||
this.settings = settings;
|
||||
|
||||
save();
|
||||
sink.emitNext(settings, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
|
||||
}
|
||||
|
||||
public void restore() {
|
||||
@@ -232,7 +230,6 @@ public class SettingsService {
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
save();
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
public void check(Settings settings) throws ValidationException {
|
||||
|
||||
@@ -19,7 +19,7 @@ import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.exception.VripperException;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable;
|
||||
import tn.mnlr.vripper.tasks.LeaveThanksRunnable;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
@@ -51,8 +51,12 @@ public class VGAuthService {
|
||||
this.cm = cm;
|
||||
this.settingsService = settingsService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate());
|
||||
this.eventBus = eventBus;
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.subscribe(e -> authenticate());
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
|
||||
@@ -68,8 +68,6 @@ public class MultiPostScanParser {
|
||||
}
|
||||
|
||||
try {
|
||||
//
|
||||
// System.out.println(EntityUtils.toString(response.getEntity()));
|
||||
factory
|
||||
.newSAXParser()
|
||||
.parse(
|
||||
|
||||
+5
-11
@@ -1,9 +1,9 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.download.PendingQueue;
|
||||
import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
@@ -31,11 +31,11 @@ public class AddPostRunnable implements Runnable {
|
||||
private final DataService dataService;
|
||||
private final MetadataService metadataService;
|
||||
private final SettingsService settingsService;
|
||||
private final PendingQueue pendingQueue;
|
||||
private final VGAuthService VGAuthService;
|
||||
private final LogEvent logEvent;
|
||||
private final ILogEventRepository eventRepository;
|
||||
private final String link;
|
||||
private final DownloadService downloadService;
|
||||
|
||||
public AddPostRunnable(String postId, String threadId) {
|
||||
this.postId = postId;
|
||||
@@ -43,7 +43,7 @@ public class AddPostRunnable implements Runnable {
|
||||
this.dataService = SpringContext.getBean(DataService.class);
|
||||
this.metadataService = SpringContext.getBean(MetadataService.class);
|
||||
this.settingsService = SpringContext.getBean(SettingsService.class);
|
||||
this.pendingQueue = SpringContext.getBean(PendingQueue.class);
|
||||
this.downloadService = SpringContext.getBean(DownloadService.class);
|
||||
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
|
||||
this.eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
link =
|
||||
@@ -112,13 +112,7 @@ public class AddPostRunnable implements Runnable {
|
||||
if (settingsService.getSettings().getAutoStart()) {
|
||||
log.debug("Auto start downloads option is enabled");
|
||||
post.setStatus(Status.PENDING);
|
||||
try {
|
||||
pendingQueue.enqueue(post, images);
|
||||
} catch (InterruptedException e) {
|
||||
log.warn("Interruption was caught");
|
||||
Thread.currentThread().interrupt();
|
||||
return;
|
||||
}
|
||||
downloadService.enqueue(post, images);
|
||||
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
|
||||
} else {
|
||||
post.setStatus(Status.STOPPED);
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.NameValuePair;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.lang.NonNull;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
+2
-2
@@ -12,8 +12,8 @@ import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostItem;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.tasks.LinkScanRunnable;
|
||||
import tn.mnlr.vripper.web.restendpoints.domain.*;
|
||||
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
|
||||
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
|
||||
|
||||
@@ -161,4 +161,11 @@
|
||||
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
|
||||
referencedColumnNames="ID" referencedTableName="POST"/>
|
||||
</changeSet>
|
||||
<changeSet id="1620590428" author="death-claw">
|
||||
<addColumn tableName="POST">
|
||||
<column name="ADDED_ON" type="TIMESTAMP" defaultValueComputed="CURRENT_TIMESTAMP">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
</addColumn>
|
||||
</changeSet>
|
||||
</databaseChangeLog>
|
||||
|
||||
Reference in New Issue
Block a user