Enhance download queue logic

This commit is contained in:
death-claw
2021-05-09 22:39:22 +01:00
parent 90af03d68f
commit 53fd14935b
23 changed files with 212 additions and 263 deletions
+1 -1
View File
@@ -59,7 +59,7 @@
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.22</version>
<version>2.24</version>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
@@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.DataService;
import java.time.LocalDateTime;
@Slf4j
public class DownloadRunnable implements Runnable {
public class DownloadJobWrapper implements Runnable {
private final DownloadService downloadService;
private final DataService dataService;
@@ -22,7 +22,7 @@ public class DownloadRunnable implements Runnable {
private final DownloadJob downloadJob;
public DownloadRunnable(final DownloadJob downloadJob) {
public DownloadJobWrapper(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
connectionService = SpringContext.getBean(ConnectionService.class);
@@ -11,10 +11,14 @@ import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@@ -27,41 +31,38 @@ public class DownloadService {
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
private final List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
private final List<DownloadJob> pending = new ArrayList<>();
private final PendingQueue pendingQueue;
private final SettingsService settings;
private final SettingsService settingsService;
private final DataService dataService;
private final PostService postService;
private final List<Host> hosts;
private boolean pauseQ = false;
private Thread executionThread;
private Thread pollThread;
@Autowired
public DownloadService(
PendingQueue pendingQueue,
SettingsService settings,
SettingsService settingsService,
DataService dataService,
PostService postService) {
this.pendingQueue = pendingQueue;
this.settings = settings;
PostService postService,
List<Host> hosts) {
this.settingsService = settingsService;
this.dataService = dataService;
this.postService = postService;
this.hosts = hosts;
}
@PostConstruct
private void init() {
executionThread = new Thread(this::start, "Executor thread");
pollThread = new Thread(this::poll, "Polling thread");
pollThread = new Thread(this::start, "Polling thread");
pollThread.start();
executionThread.start();
}
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
executionThread.interrupt();
pollThread.interrupt();
executor.shutdown();
dataService
@@ -76,7 +77,7 @@ public class DownloadService {
private void stopRunning(@NonNull String postId) {
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = executing.iterator();
Iterator<DownloadJob> iterator = running.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
@@ -125,18 +126,15 @@ public class DownloadService {
post.setStatus(Status.PENDING);
dataService.updatePostStatus(post.getStatus(), post.getId());
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
try {
pendingQueue.put(post, image);
} catch (InterruptedException e) {
log.warn("Thread was interrupted", e);
Thread.currentThread().interrupt();
}
}
enqueue(post, images);
}
private boolean isPending(String postId) {
return pendingQueue.isPending(postId);
return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private boolean isRunning(String postId) {
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private void stop(String postId) {
@@ -149,7 +147,7 @@ public class DownloadService {
if (FINISHED.contains(post.getStatus())) {
return;
}
pendingQueue.stop(post);
pending.removeIf(p -> p.getPost().equals(post));
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
@@ -161,16 +159,12 @@ public class DownloadService {
private boolean canRun(Host host) {
boolean canRun;
AtomicInteger count = threadCount.get(host);
if (count == null) {
threadCount.put(host, new AtomicInteger(0));
}
int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
canRun =
threadCount.get(host).get() < settings.getSettings().getMaxThreads()
&& (settings.getSettings().getMaxTotalThreads() == 0
? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE
: threadCount.values().stream().mapToInt(AtomicInteger::get).sum()
< settings.getSettings().getMaxTotalThreads());
threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
&& (settingsService.getSettings().getMaxTotalThreads() == 0
? totalRunning < MAX_POOL_SIZE
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
if (canRun && !pauseQ) {
threadCount.get(host).incrementAndGet();
return true;
@@ -178,59 +172,102 @@ public class DownloadService {
return false;
}
private void poll() {
while (!Thread.interrupted()) {
try {
List<DownloadJob> peek = pendingQueue.peek();
for (DownloadJob downloadJob : peek) {
if (canRun(downloadJob.getImage().getHost())) {
executionQueue.offer(downloadJob);
pendingQueue.remove(downloadJob);
private Map<Host, Integer> candidateCount() {
HashMap<Host, Integer> map = new HashMap<>();
hosts.forEach(
h -> {
AtomicInteger count = threadCount.get(h);
if (count == null) {
count = new AtomicInteger(0);
threadCount.put(h, count);
}
}
synchronized (threadCount) {
threadCount.wait(2_000);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
});
return map;
}
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount, int max) {
int i = 0;
List<DownloadJob> candidates = new ArrayList<>();
for (DownloadJob downloadJob : pending) {
Host host = downloadJob.getImage().getHost();
Integer maxPerHost = candidateCount.get(host);
if (maxPerHost > 0 && i < max) {
candidates.add(downloadJob);
candidateCount.put(host, maxPerHost + 1);
i++;
}
if (i >= max) {
break;
}
}
return candidates;
}
public void enqueue(Post post, Collection<Image> imageList) {
synchronized (this) {
for (Image image : imageList) {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob =
new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
pending.add(downloadJob);
}
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
this.notify();
}
}
private void start() {
while (!Thread.interrupted()) {
try {
push(executionQueue.take());
synchronized (this) {
List<DownloadJob> accepted = new ArrayList<>();
List<DownloadJob> candidates = getCandidates(candidateCount(), MAX_POOL_SIZE);
candidates.forEach(
c -> {
if (canRun(c.getImage().getHost())) {
accepted.add(c);
}
});
pending.removeAll(accepted);
accepted.forEach(this::push);
accepted.clear();
this.wait();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
} catch (Exception e) {
log.error("Execution Service failed", e);
break;
}
}
}
private void push(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new DownloadRunnable(downloadJob));
executing.add(downloadJob);
executor.execute(new DownloadJobWrapper(downloadJob));
running.add(downloadJob);
}
public synchronized void afterJobFinish(DownloadJob downloadJob) {
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
if (count == 0) {
dataService.finishPost(downloadJob.getPost());
}
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
executing.remove(downloadJob);
synchronized (threadCount) {
threadCount.notify();
public void afterJobFinish(DownloadJob downloadJob) {
synchronized (this) {
running.remove(downloadJob);
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
if (!isPending(downloadJob.getPost().getPostId())
&& !isRunning(downloadJob.getPost().getPostId())) {
dataService.finishPost(downloadJob.getPost());
}
this.notify();
}
}
public int pendingCount() {
return pending.size();
}
public int runningCount() {
return executing.size();
return running.size();
}
}
@@ -1,118 +0,0 @@
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
@Service
@Slf4j
public class PendingQueue {
private final DataService dataService;
private final SettingsService settingsService;
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ =
new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
@Autowired
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
this.dataService = dataService;
this.settingsService = settingsService;
this.hosts = hosts;
}
@PostConstruct
private void init() {
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
}
public void put(Post post, Image image) throws InterruptedException {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob =
new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
checkKey(post.getPostId());
toBeExecuted.get(post.getPostId()).incrementAndGet();
}
private synchronized void checkKey(String postId) {
if (!toBeExecuted.containsKey(postId)) {
toBeExecuted.put(postId, new AtomicInteger(0));
}
}
public void remove(final DownloadJob downloadJob) {
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
}
public List<DownloadJob> peek() {
List<DownloadJob> downloadJobs = new ArrayList<>();
if (hosts.size() == 0) {
return downloadJobs;
}
for (Host host : hosts) {
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
if (downloadJob != null) {
downloadJobs.add(downloadJob);
}
}
}
return downloadJobs;
}
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
for (Image image : images) {
put(post, image);
}
}
public int size() {
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
}
public void stop(Post post) {
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
entry.getValue().removeIf(predicate);
decrement(post.getPostId());
}
}
public boolean isPending(String postId) {
return toBeExecuted.containsKey(postId);
}
public synchronized int decrement(String postId) {
AtomicInteger counter = toBeExecuted.get(postId);
if (counter == null) {
return 0;
}
int count = counter.decrementAndGet();
if (count == 0) {
toBeExecuted.remove(postId);
}
return count;
}
}
@@ -28,6 +28,7 @@ public class Event<T> {
LOG_EVENT_REMOVE,
VG_USER,
GLOBAL_STATE,
BYTES_PER_SECOND
BYTES_PER_SECOND,
SETTINGS_UPDATE
}
}
@@ -7,6 +7,7 @@ import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@@ -49,6 +50,8 @@ public class Post {
private Metadata metadata;
private LocalDateTime addedOn;
public Post(
String title,
String url,
@@ -65,6 +68,7 @@ public class Post {
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
addedOn = LocalDateTime.now();
}
@Override
@@ -13,6 +13,7 @@ import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -37,7 +38,7 @@ public class PostRepository implements IPostRepository {
public Post save(Post post) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
@@ -52,7 +53,8 @@ public class PostRepository implements IPostRepository {
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
post.getUrl());
post.getUrl(),
Timestamp.valueOf(post.getAddedOn()));
post.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return post;
@@ -198,6 +200,7 @@ class PostRowMapper implements RowMapper<Post> {
if ((previews = rs.getString("post.PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
Long metadataId = rs.getLong("metadata.POST_ID_REF");
if (!rs.wasNull()) {
@@ -1,7 +0,0 @@
package tn.mnlr.vripper.listener;
import reactor.core.publisher.Flux;
public interface DataEventListener<T> {
Flux<T> getDataFlux();
}
@@ -1,7 +0,0 @@
package tn.mnlr.vripper.listener;
import reactor.core.publisher.Sinks.EmitFailureHandler;
public class EmitHandler {
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
}
@@ -17,8 +17,9 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
import reactor.core.publisher.Flux;
import tn.mnlr.vripper.download.DownloadJob;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PreDestroy;
@@ -41,22 +42,25 @@ public class ConnectionService {
private int connectionTimeout;
private int maxAttempts;
public ConnectionService(SettingsService settingsService) {
public ConnectionService(EventBus eventBus, SettingsService settingsService) {
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
maxAttempts = settingsService.getSettings().getMaxAttempts();
Flux<Settings> settingsFlux = settingsService.getSettingsFlux();
disposable =
settingsFlux.subscribe(
settings -> {
if (connectionTimeout != settings.getConnectionTimeout()) {
connectionTimeout = settings.getConnectionTimeout();
buildRequestConfig();
}
if (maxAttempts != settings.getMaxAttempts()) {
maxAttempts = settings.getMaxAttempts();
buildRetryPolicy();
}
});
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.map(e -> ((Settings) e.getData()))
.subscribe(
settings -> {
if (connectionTimeout != settings.getConnectionTimeout()) {
connectionTimeout = settings.getConnectionTimeout();
buildRequestConfig();
}
if (maxAttempts != settings.getMaxAttempts()) {
maxAttempts = settings.getMaxAttempts();
buildRetryPolicy();
}
});
buildRequestConfig();
buildRetryPolicy();
@@ -6,7 +6,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.download.PendingQueue;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.services.domain.GlobalState;
@@ -15,7 +14,6 @@ import tn.mnlr.vripper.services.domain.GlobalState;
@EnableScheduling
public class GlobalStateService {
private final PendingQueue pendingQueue;
private final DownloadService downloadService;
private final DataService dataService;
private final EventBus eventBus;
@@ -23,11 +21,7 @@ public class GlobalStateService {
@Autowired
public GlobalStateService(
PendingQueue pendingQueue,
DownloadService downloadService,
DataService dataService,
EventBus eventBus) {
this.pendingQueue = pendingQueue;
DownloadService downloadService, DataService dataService, EventBus eventBus) {
this.downloadService = downloadService;
this.dataService = dataService;
this.eventBus = eventBus;
@@ -37,7 +31,9 @@ public class GlobalStateService {
private void interval() {
GlobalState newGlobalState =
new GlobalState(
downloadService.runningCount(), pendingQueue.size(), dataService.countErrorImages());
downloadService.runningCount(),
downloadService.pendingCount(),
dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
@@ -12,12 +12,16 @@ import org.apache.http.util.EntityUtils;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import reactor.core.Disposable;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable;
import tn.mnlr.vripper.tasks.MetadataRunnable;
import javax.annotation.PreDestroy;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -38,19 +42,33 @@ public class MetadataService {
private final XpathService xpathService;
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
private final ThreadPoolService threadPoolService;
private final Disposable disposable;
public MetadataService(
ConnectionService cm,
VGAuthService VGAuthService,
HtmlProcessorService htmlProcessorService,
XpathService xpathService,
ThreadPoolService threadPoolService) {
ThreadPoolService threadPoolService,
EventBus eventBus) {
this.cm = cm;
this.VGAuthService = VGAuthService;
this.htmlProcessorService = htmlProcessorService;
this.xpathService = xpathService;
this.threadPoolService = threadPoolService;
cache = Caffeine.newBuilder().expireAfterWrite(30, TimeUnit.MINUTES).build(this::fetchMetadata);
disposable =
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> this.cache.invalidateAll());
}
@PreDestroy
private void destroy() {
if (disposable != null) {
disposable.dispose();
}
}
public Metadata get(Post post) throws ExecutionException {
@@ -5,13 +5,17 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.domain.MultiPostScanParser;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable;
import tn.mnlr.vripper.tasks.AddPostRunnable;
import tn.mnlr.vripper.tasks.AddQueuedRunnable;
import javax.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -24,12 +28,14 @@ public class PostService {
private final ThreadPoolService threadPoolService;
private final MetadataService metadataService;
private final LoadingCache<Queued, MultiPostScanResult> cache;
private final Disposable disposable;
@Autowired
public PostService(
DataService dataService,
ThreadPoolService threadPoolService,
MetadataService metadataService) {
MetadataService metadataService,
EventBus eventBus) {
this.dataService = dataService;
this.threadPoolService = threadPoolService;
this.metadataService = metadataService;
@@ -37,6 +43,18 @@ public class PostService {
Caffeine.newBuilder()
.expireAfterWrite(5, TimeUnit.MINUTES)
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
disposable =
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> this.cache.invalidateAll());
}
@PreDestroy
private void destroy() {
if (disposable != null) {
disposable.dispose();
}
}
public void stopFetchingMetadata(Post post) {
@@ -12,11 +12,10 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.ValidationException;
import tn.mnlr.vripper.listener.EmitHandler;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
@@ -46,7 +45,7 @@ public class SettingsService {
private final Set<String> proxies = new HashSet<>();
private Sinks.Many<Settings> sink = Sinks.many().multicast().onBackpressureBuffer();
private final EventBus eventBus;
@Getter private Settings settings = new Settings();
@@ -54,7 +53,10 @@ public class SettingsService {
private Resource defaultProxies;
public SettingsService(
@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
@Value("${base.dir}") String baseDir,
@Value("${base.dir.name}") String baseDirName,
EventBus eventBus) {
this.eventBus = eventBus;
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -64,7 +66,7 @@ public class SettingsService {
private void init() {
loadViperProxies();
restore();
sink.emitNext(settings, EmitHandler.RETRY);
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
private void loadViperProxies() {
@@ -93,10 +95,6 @@ public class SettingsService {
proxies.add("https://vipergirls.to");
}
Flux<Settings> getSettingsFlux() {
return sink.asFlux();
}
public List<String> getProxies() {
return new ArrayList<>(proxies);
}
@@ -116,7 +114,7 @@ public class SettingsService {
this.settings = settings;
save();
sink.emitNext(settings, EmitHandler.RETRY);
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
public void restore() {
@@ -232,7 +230,6 @@ public class SettingsService {
@PreDestroy
private void destroy() {
save();
sink.emitComplete(EmitHandler.RETRY);
}
public void check(Settings settings) throws ValidationException {
@@ -19,7 +19,7 @@ import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable;
import tn.mnlr.vripper.tasks.LeaveThanksRunnable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@@ -51,8 +51,12 @@ public class VGAuthService {
this.cm = cm;
this.settingsService = settingsService;
this.threadPoolService = threadPoolService;
disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate());
this.eventBus = eventBus;
disposable =
eventBus
.flux()
.filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> authenticate());
}
@PostConstruct
@@ -68,8 +68,6 @@ public class MultiPostScanParser {
}
try {
//
// System.out.println(EntityUtils.toString(response.getEntity()));
factory
.newSAXParser()
.parse(
@@ -1,9 +1,9 @@
package tn.mnlr.vripper.services.domain.tasks;
package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.download.PendingQueue;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.LogEvent;
@@ -31,11 +31,11 @@ public class AddPostRunnable implements Runnable {
private final DataService dataService;
private final MetadataService metadataService;
private final SettingsService settingsService;
private final PendingQueue pendingQueue;
private final VGAuthService VGAuthService;
private final LogEvent logEvent;
private final ILogEventRepository eventRepository;
private final String link;
private final DownloadService downloadService;
public AddPostRunnable(String postId, String threadId) {
this.postId = postId;
@@ -43,7 +43,7 @@ public class AddPostRunnable implements Runnable {
this.dataService = SpringContext.getBean(DataService.class);
this.metadataService = SpringContext.getBean(MetadataService.class);
this.settingsService = SpringContext.getBean(SettingsService.class);
this.pendingQueue = SpringContext.getBean(PendingQueue.class);
this.downloadService = SpringContext.getBean(DownloadService.class);
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
this.eventRepository = SpringContext.getBean(ILogEventRepository.class);
link =
@@ -112,13 +112,7 @@ public class AddPostRunnable implements Runnable {
if (settingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
try {
pendingQueue.enqueue(post, images);
} catch (InterruptedException e) {
log.warn("Interruption was caught");
Thread.currentThread().interrupt();
return;
}
downloadService.enqueue(post, images);
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.domain.tasks;
package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import tn.mnlr.vripper.SpringContext;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.domain.tasks;
package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.domain.tasks;
package tn.mnlr.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import org.springframework.lang.NonNull;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.domain.tasks;
package tn.mnlr.vripper.tasks;
import lombok.Getter;
import lombok.NonNull;
@@ -12,8 +12,8 @@ import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.domain.MultiPostItem;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable;
import tn.mnlr.vripper.tasks.AddPostRunnable;
import tn.mnlr.vripper.tasks.LinkScanRunnable;
import tn.mnlr.vripper.web.restendpoints.domain.*;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
@@ -161,4 +161,11 @@
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
referencedColumnNames="ID" referencedTableName="POST"/>
</changeSet>
<changeSet id="1620590428" author="death-claw">
<addColumn tableName="POST">
<column name="ADDED_ON" type="TIMESTAMP" defaultValueComputed="CURRENT_TIMESTAMP">
<constraints nullable="false"/>
</column>
</addColumn>
</changeSet>
</databaseChangeLog>