Compare commits

..
4 Commits
Author SHA1 Message Date
death-claw 37900cac49 v3.0.2 2020-08-15 15:38:53 +01:00
death-claw be57c76d93 Fix bugs with download queue 2020-08-15 15:36:47 +01:00
death-claw 8b52ad7858 v3.0.1 2020-08-03 18:48:45 +01:00
death-claw a0976bde02 Fix bugs with download queue 2020-08-03 18:43:42 +01:00
25 changed files with 143 additions and 129 deletions
+8
View File
@@ -1,5 +1,13 @@
# Changelog
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
+1 -1
View File
@@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.0.2</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "3.0.0",
"version": "3.0.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "3.0.0",
"version": "3.0.2",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.0.2</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.0.2</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
@@ -6,17 +6,14 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
@SpringBootApplication
@Slf4j
public class VripperApplication {
public static final RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
.handleIf(e -> !(e instanceof InterruptedException))
.withDelay(1, 3, ChronoUnit.SECONDS)
.withMaxAttempts(5)
.abortOn(Collections.singletonList(InterruptedException.class))
.onFailedAttempt(e -> log.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
public static void main(String[] args) {
@@ -27,6 +24,5 @@ public class VripperApplication {
log.error("Failed to run the application", e);
}
}
}
@@ -18,6 +18,7 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.q.DownloadJob;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.*;
@@ -79,17 +80,18 @@ abstract public class Host {
return url.contains(getLookup());
}
public void download(final Post post, final Image image, final ImageFileData imageFileData) throws DownloadException, InterruptedException {
public void download(final Post post, final Image image, final ImageFileData imageFileData, DownloadJob downloadJob) throws DownloadException, InterruptedException {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
try {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
@@ -127,16 +129,25 @@ abstract public class Host {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (downloadJob.isStopped()) {
return;
}
File destinationFolder;
synchronized (LOCK) {
if (post.getPostFolderName() == null) {
pathService.createDefaultPostFolder(post);
Post updatedPost = dataService.findPostById(post.getId()).orElseThrow();
if (updatedPost.getPostFolderName() == null) {
pathService.createDefaultPostFolder(updatedPost);
}
destinationFolder = pathService.getDownloadDestinationFolder(post);
authService.leaveThanks(post);
destinationFolder = pathService.getDownloadDestinationFolder(updatedPost);
authService.leaveThanks(updatedPost);
}
File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
if (downloadJob.isStopped()) {
return;
}
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
@@ -145,7 +156,7 @@ abstract public class Host {
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !downloadJob.isStopped()) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
@@ -153,22 +164,25 @@ abstract public class Host {
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
} finally {
if (image.getCurrent() == image.getTotal()) {
image.setStatus(Status.COMPLETE);
} else {
image.setStatus(Status.ERROR);
if (downloadJob.isStopped()) {
return;
}
dataService.updateImageStatus(image.getStatus(), image.getId());
}
File finalName = checkImageTypeAndRename(post, outputFile, imageFileData.getImageName(), image.getIndex());
File finalName = checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, imageFileData.getImageName(), image.getIndex());
imageFileData.setFileName(finalName.getName());
}
} catch (Exception e) {
if (Thread.interrupted() || imageFileData.getImageRequest().isAborted()) {
throw new InterruptedException("Download was interrupted");
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal()) {
image.setStatus(Status.COMPLETE);
} else if (downloadJob.isStopped()) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
downloadJob.done();
}
}
@@ -299,8 +313,8 @@ abstract public class Host {
this.headers = headers;
}
private Document document;
private Header[] headers;
private final Document document;
private final Header[] headers;
}
@Override
@@ -14,8 +14,6 @@ public interface IImageRepository extends IRepository {
List<Image> findByPostId(String postId);
Integer countRemaining();
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
@@ -67,14 +67,6 @@ public class ImageRepository implements IImageRepository {
);
}
@Override
public Integer countRemaining() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.TOTAL = 0 OR image.TOTAL <> image.CURRENT",
Integer.class
);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
@@ -3,38 +3,42 @@ package tn.mnlr.vripper.q;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.function.CheckedRunnable;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
import java.util.Objects;
@Slf4j
public class DownloadJob implements CheckedRunnable {
private final DataService dataService;
@Getter
private final Image image;
@Getter
private Image image;
@Getter
private Post post;
private final Post post;
@Getter
private final ImageFileData imageFileData = new ImageFileData();
@Getter
private boolean stopped = false;
@Getter
private boolean finished = false;
DownloadJob(Post post, Image image) {
this.image = image;
this.post = post;
dataService = SpringContext.getBean(DataService.class);
}
@Override
public void run() throws Exception {
if (stopped) {
done();
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
image.getHost().download(post, image, imageFileData);
image.getHost().download(post, image, imageFileData, this);
}
@Override
@@ -51,8 +55,11 @@ public class DownloadJob implements CheckedRunnable {
return Objects.hash(image, post);
}
public void refresh() {
post = dataService.findPostById(post.getId()).orElseThrow();
image = dataService.findImageById(image.getId()).orElseThrow();
public void stop() {
this.stopped = true;
}
public void done() {
finished = true;
}
}
@@ -28,18 +28,11 @@ public class ExecuteRunnable implements Runnable {
@Override
public void run() {
mutexService.createPostLock(downloadJob.getPost().getPostId());
ReentrantLock mutex = mutexService.getPostLock(downloadJob.getPost().getPostId());
mutex.lock();
downloadJob.refresh();
executionService.beforeJobStart(downloadJob.getPost().getPostId());
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> {
if (e.getFailure() instanceof InterruptedException || e.getFailure().getCause() instanceof InterruptedException) {
log.debug("Job successfully interrupted");
return;
}
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
@@ -15,9 +15,7 @@ import tn.mnlr.vripper.services.post.PostService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -31,8 +29,7 @@ public class ExecutionService {
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
private final Map<DownloadJob, Future<?>> futures = new ConcurrentHashMap<>();
private final Map<String, AtomicInteger> downloadCount = new ConcurrentHashMap<>();
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
private final PendingQ pendingQ;
private final AppSettingsService settings;
@@ -75,18 +72,25 @@ public class ExecutionService {
}
private void stopRunning(@NonNull String postId) {
futures
.entrySet()
.stream()
.filter(e -> e.getKey().getImage().getPostId().equals(postId))
.forEach(e -> {
e.getValue().cancel(true);
if (e.getKey().getImageFileData().getImageRequest() != null) {
e.getKey().getImageFileData().getImageRequest().abort();
}
e.getKey().getImage().setStatus(Status.STOPPED);
dataService.updateImageStatus(e.getKey().getImage().getStatus(), e.getKey().getImage().getId());
});
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = executing.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(200);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> posIds) {
@@ -106,7 +110,7 @@ public class ExecutionService {
}
private void restart(@NonNull String postId) {
if (isRunning(postId) || isPending(postId)) {
if (isPending(postId)) {
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
@@ -132,11 +136,6 @@ public class ExecutionService {
return pendingQ.isPending(postId);
}
public boolean isRunning(@NonNull final String postId) {
AtomicInteger runningCount = downloadCount.get(postId);
return runningCount != null && runningCount.get() > 0;
}
private void stop(String postId) {
try {
pauseQ = true;
@@ -147,9 +146,10 @@ public class ExecutionService {
if (FINISHED.contains(post.getStatus())) {
return;
}
pendingQ.remove(post);
pendingQ.stop(post);
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
postService.stopFetchingMetadata(post);
} finally {
pauseQ = false;
@@ -205,37 +205,25 @@ public class ExecutionService {
}
private void push(DownloadJob downloadJob) {
ExecuteRunnable runnable = new ExecuteRunnable(downloadJob);
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
futures.put(downloadJob, executor.submit(runnable));
}
public void beforeJobStart(String postId) {
checkKeyRunningPosts(postId);
downloadCount.get(postId).incrementAndGet();
}
private synchronized void checkKeyRunningPosts(@NonNull final String postId) {
if (!downloadCount.containsKey(postId)) {
downloadCount.put(postId, new AtomicInteger(0));
}
executor.execute(new ExecuteRunnable(downloadJob));
executing.add(downloadJob);
}
public synchronized void afterJobFinish(DownloadJob downloadJob) {
int count = downloadCount.get(downloadJob.getPost().getPostId()).decrementAndGet();
if (count == 0 && !pendingQ.isPending(downloadJob.getPost().getPostId())) {
downloadCount.remove(downloadJob.getPost().getPostId());
int count = pendingQ.decrement(downloadJob.getPost().getPostId());
if (count == 0) {
dataService.finishPost(downloadJob.getPost());
mutexService.removePostLock(downloadJob.getPost().getPostId());
}
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
futures.remove(downloadJob);
executing.remove(downloadJob);
synchronized (threadCount) {
threadCount.notify();
}
}
public int runningCount() {
return futures.size();
return executing.size();
}
}
@@ -14,6 +14,8 @@ import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
@Service
@Slf4j
@@ -24,6 +26,7 @@ public class PendingQ {
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
@Autowired
public PendingQ(DataService dataService, AppSettingsService appSettingsService, List<Host> hosts) {
@@ -44,6 +47,14 @@ public class PendingQ {
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob = new DownloadJob(post, image);
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
checkKey(post.getPostId());
toBeExecuted.get(post.getPostId()).incrementAndGet();
}
private synchronized void checkKey(String postId) {
if (!toBeExecuted.containsKey(postId)) {
toBeExecuted.put(postId, new AtomicInteger(0));
}
}
public void remove(final DownloadJob downloadJob) {
@@ -74,17 +85,31 @@ public class PendingQ {
}
public int size() {
return pendingQ.values().stream().mapToInt(BlockingDeque::size).sum();
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
}
public void remove(Post post) {
public void stop(Post post) {
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
entry.getValue().removeIf(next -> next.getImage().getPostId().equals(post.getPostId()));
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
entry.getValue().removeIf(predicate);
decrement(post.getPostId());
}
dataService.finishPost(post);
}
public boolean isPending(String postId) {
return pendingQ.values().stream().flatMap(Collection::stream).anyMatch(e -> e.getPost().getPostId().equals(postId));
return toBeExecuted.containsKey(postId);
}
public synchronized int decrement(String postId) {
AtomicInteger counter = toBeExecuted.get(postId);
if (counter == null) {
return 0;
}
int count = counter.decrementAndGet();
if (count == 0) {
toBeExecuted.remove(postId);
}
return count;
}
}
@@ -169,10 +169,6 @@ public class DataService {
}
public long countRemainingImages() {
return imageRepository.countRemaining();
}
public long countErrorImages() {
return imageRepository.countError();
}
@@ -8,13 +8,11 @@ import java.util.Objects;
public class GlobalState {
private final long running;
private final long queued;
private final long remaining;
private final long error;
GlobalState(long running, long queued, long remaining, long error) {
GlobalState(long running, long remaining, long error) {
this.running = running;
this.queued = queued;
this.remaining = remaining;
this.error = error;
}
@@ -24,11 +22,11 @@ public class GlobalState {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GlobalState that = (GlobalState) o;
return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
return running == that.running && remaining == that.remaining && error == that.error;
}
@Override
public int hashCode() {
return Objects.hash(running, queued, remaining, error);
return Objects.hash(running, remaining, error);
}
}
@@ -35,7 +35,6 @@ public class GlobalStateService {
GlobalState newGlobalState = new GlobalState(
executionService.runningCount(),
pendingQ.size(),
dataService.countRemainingImages(),
dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-ui",
"version": "3.0.0",
"version": "3.0.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-ui",
"version": "3.0.0",
"version": "3.0.2",
"scripts": {
"ng": "ng",
"start": "ng serve",
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.0.2</version>
</parent>
<artifactId>vripper-ui</artifactId>
<name>vripper-ui</name>
@@ -1,3 +1,4 @@
export class GlobalState {
constructor(public running: number, public queued: number, public remaining: number, public error: number) {}
constructor(public running: number, public remaining: number, public error: number) {
}
}
@@ -4,7 +4,6 @@
<span *ngIf="downloadSpeed$ | async as speed">{{ speed.speed + '/s' }}</span>
<ng-container *ngIf="globalState$ | async as state">
<span>Downloading: {{ state!.running }}</span>
<span>Pending: {{ state!.queued }}</span>
<span>Remaining: {{ state!.remaining }}</span>
<span>Error: {{ state!.error }}</span>
</ng-container>
@@ -16,7 +16,7 @@ export class StatusBarComponent implements OnInit, OnDestroy {
}
downloadSpeed$: Subject<DownloadSpeed> = new BehaviorSubject(new DownloadSpeed('0 B'));
globalState$: Subject<GlobalState> = new BehaviorSubject(new GlobalState(0, 0, 0, 0));
globalState$: Subject<GlobalState> = new BehaviorSubject(new GlobalState(0, 0, 0));
selected$: Subject<number> = new BehaviorSubject(0);
subscriptions: Subscription[] = [];
@@ -2,5 +2,5 @@ export const environment = {
production: true,
localhost: `${window.location.protocol}//${window.location.host}`,
ws: `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}`,
version: '3.0.0'
version: '3.0.2'
};
+1 -1
View File
@@ -6,7 +6,7 @@ export const environment = {
production: false,
localhost: 'http://localhost:8080',
ws: 'ws://localhost:8080',
version: '3.0.0'
version: '3.0.2'
};
/*