v3.0.1

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## [3.0.1] - 2020-08-03
+### Changed
+- Fix bugs with download queue
+
 ## [3.0.0] - 2020-08-03
 ### Changed
 - Major rewrites

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>3.0.0</version>
+        <version>3.0.1</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -129,11 +129,12 @@ abstract public class Host {
                 }
                 File destinationFolder;
                 synchronized (LOCK) {
-                    if (post.getPostFolderName() == null) {
-                        pathService.createDefaultPostFolder(post);
+                    Post updatedPost = dataService.findPostById(post.getId()).orElseThrow();
+                    if (updatedPost.getPostFolderName() == null) {
+                        pathService.createDefaultPostFolder(updatedPost);
                     }
-                    destinationFolder = pathService.getDownloadDestinationFolder(post);
-                    authService.leaveThanks(post);
+                    destinationFolder = pathService.getDownloadDestinationFolder(updatedPost);
+                    authService.leaveThanks(updatedPost);
                 }
                 File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
                 try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
@@ -161,7 +162,7 @@ abstract public class Host {
                     }
                     dataService.updateImageStatus(image.getStatus(), image.getId());
                 }
-                File finalName = checkImageTypeAndRename(post, outputFile, imageFileData.getImageName(), image.getIndex());
+                File finalName = checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, imageFileData.getImageName(), image.getIndex());
                 imageFileData.setFileName(finalName.getName());
             }
         } catch (Exception e) {
@@ -299,8 +300,8 @@ abstract public class Host {
             this.headers = headers;
         }
 
-        private Document document;
-        private Header[] headers;
+        private final Document document;
+        private final Header[] headers;
     }
 
     @Override

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IImageRepository.java

@@ -14,8 +14,6 @@ public interface IImageRepository extends IRepository {
 
     List<Image> findByPostId(String postId);
 
-    Integer countRemaining();
-
     Integer countError();
 
     List<Image> findByPostIdAndIsNotCompleted(String postId);

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/ImageRepository.java

@@ -67,14 +67,6 @@ public class ImageRepository implements IImageRepository {
         );
     }
 
-    @Override
-    public Integer countRemaining() {
-        return jdbcTemplate.queryForObject(
-                "SELECT COUNT(*) FROM IMAGE AS image WHERE image.TOTAL = 0 OR image.TOTAL <> image.CURRENT",
-                Integer.class
-        );
-    }
-
     @Override
     public Integer countError() {
         return jdbcTemplate.queryForObject(

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java

@@ -3,23 +3,19 @@ package tn.mnlr.vripper.q;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.failsafe.function.CheckedRunnable;
-import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.jpa.domain.Image;
 import tn.mnlr.vripper.jpa.domain.Post;
-import tn.mnlr.vripper.services.DataService;
 
 import java.util.Objects;
 
 @Slf4j
 public class DownloadJob implements CheckedRunnable {
 
-    private final DataService dataService;
+    @Getter
+    private final Image image;
 
     @Getter
-    private Image image;
-
-    @Getter
-    private Post post;
+    private final Post post;
 
     @Getter
     private final ImageFileData imageFileData = new ImageFileData();
@@ -27,7 +23,6 @@ public class DownloadJob implements CheckedRunnable {
     DownloadJob(Post post, Image image) {
         this.image = image;
         this.post = post;
-        dataService = SpringContext.getBean(DataService.class);
     }
 
     @Override
@@ -50,9 +45,4 @@ public class DownloadJob implements CheckedRunnable {
     public int hashCode() {
         return Objects.hash(image, post);
     }
-
-    public void refresh() {
-        post = dataService.findPostById(post.getId()).orElseThrow();
-        image = dataService.findImageById(image.getId()).orElseThrow();
-    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecuteRunnable.java

@@ -32,8 +32,6 @@ public class ExecuteRunnable implements Runnable {
         mutexService.createPostLock(downloadJob.getPost().getPostId());
         ReentrantLock mutex = mutexService.getPostLock(downloadJob.getPost().getPostId());
         mutex.lock();
-        downloadJob.refresh();
-        executionService.beforeJobStart(downloadJob.getPost().getPostId());
         Failsafe.with(VripperApplication.retryPolicy)
                 .onFailure(e -> {
                     if (e.getFailure() instanceof InterruptedException || e.getFailure().getCause() instanceof InterruptedException) {

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -32,7 +32,7 @@ public class ExecutionService {
     private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
     private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
     private final Map<DownloadJob, Future<?>> futures = new ConcurrentHashMap<>();
-    private final Map<String, AtomicInteger> downloadCount = new ConcurrentHashMap<>();
+//    private final Map<String, AtomicInteger> downloadCount = new ConcurrentHashMap<>();
 
     private final PendingQ pendingQ;
     private final AppSettingsService settings;
@@ -106,7 +106,7 @@ public class ExecutionService {
     }
 
     private void restart(@NonNull String postId) {
-        if (isRunning(postId) || isPending(postId)) {
+        if (isPending(postId)) {
             log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
             return;
         }
@@ -132,11 +132,6 @@ public class ExecutionService {
         return pendingQ.isPending(postId);
     }
 
-    public boolean isRunning(@NonNull final String postId) {
-        AtomicInteger runningCount = downloadCount.get(postId);
-        return runningCount != null && runningCount.get() > 0;
-    }
-
     private void stop(String postId) {
         try {
             pauseQ = true;
@@ -210,21 +205,9 @@ public class ExecutionService {
         futures.put(downloadJob, executor.submit(runnable));
     }
 
-    public void beforeJobStart(String postId) {
-        checkKeyRunningPosts(postId);
-        downloadCount.get(postId).incrementAndGet();
-    }
-
-    private synchronized void checkKeyRunningPosts(@NonNull final String postId) {
-        if (!downloadCount.containsKey(postId)) {
-            downloadCount.put(postId, new AtomicInteger(0));
-        }
-    }
-
     public synchronized void afterJobFinish(DownloadJob downloadJob) {
-        int count = downloadCount.get(downloadJob.getPost().getPostId()).decrementAndGet();
-        if (count == 0 && !pendingQ.isPending(downloadJob.getPost().getPostId())) {
-            downloadCount.remove(downloadJob.getPost().getPostId());
+        int count = pendingQ.afterJobFinish(downloadJob.getPost().getPostId());
+        if (count == 0) {
             dataService.finishPost(downloadJob.getPost());
             mutexService.removePostLock(downloadJob.getPost().getPostId());
         }

vripper-server/src/main/java/tn/mnlr/vripper/q/PendingQ.java

@@ -14,6 +14,7 @@ import java.util.*;
 import java.util.concurrent.BlockingDeque;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicInteger;
 
 @Service
 @Slf4j
@@ -24,6 +25,7 @@ public class PendingQ {
     private final List<Host> hosts;
 
     private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
 
     @Autowired
     public PendingQ(DataService dataService, AppSettingsService appSettingsService, List<Host> hosts) {
@@ -44,6 +46,14 @@ public class PendingQ {
         dataService.updateImageCurrent(image.getCurrent(), image.getId());
         DownloadJob downloadJob = new DownloadJob(post, image);
         pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
+        checkKey(post.getPostId());
+        toBeExecuted.get(post.getPostId()).incrementAndGet();
+    }
+
+    private synchronized void checkKey(String postId) {
+        if (!toBeExecuted.containsKey(postId)) {
+            toBeExecuted.put(postId, new AtomicInteger(0));
+        }
     }
 
     public void remove(final DownloadJob downloadJob) {
@@ -74,7 +84,7 @@ public class PendingQ {
     }
 
     public int size() {
-        return pendingQ.values().stream().mapToInt(BlockingDeque::size).sum();
+        return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
     }
 
     public void remove(Post post) {
@@ -85,6 +95,14 @@ public class PendingQ {
     }
 
     public boolean isPending(String postId) {
-        return pendingQ.values().stream().flatMap(Collection::stream).anyMatch(e -> e.getPost().getPostId().equals(postId));
+        return toBeExecuted.containsKey(postId);
+    }
+
+    public int afterJobFinish(String postId) {
+        int count = toBeExecuted.get(postId).decrementAndGet();
+        if (count == 0) {
+            toBeExecuted.remove(postId);
+        }
+        return count;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java

@@ -169,10 +169,6 @@ public class DataService {
 
     }
 
-    public long countRemainingImages() {
-        return imageRepository.countRemaining();
-    }
-
     public long countErrorImages() {
         return imageRepository.countError();
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalState.java

@@ -8,13 +8,11 @@ import java.util.Objects;
 public class GlobalState {
 
     private final long running;
-    private final long queued;
     private final long remaining;
     private final long error;
 
-    GlobalState(long running, long queued, long remaining, long error) {
+    GlobalState(long running, long remaining, long error) {
         this.running = running;
-        this.queued = queued;
         this.remaining = remaining;
         this.error = error;
     }
@@ -24,11 +22,11 @@ public class GlobalState {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         GlobalState that = (GlobalState) o;
-        return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
+        return running == that.running && remaining == that.remaining && error == that.error;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(running, queued, remaining, error);
+        return Objects.hash(running, remaining, error);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java

@@ -35,7 +35,6 @@ public class GlobalStateService {
         GlobalState newGlobalState = new GlobalState(
                 executionService.runningCount(),
                 pendingQ.size(),
-                dataService.countRemainingImages(),
                 dataService.countErrorImages());
         if (!newGlobalState.equals(currentState)) {
             currentState = newGlobalState;

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/domain/global-state.model.ts

@@ -1,3 +1,4 @@
 export class GlobalState {
-    constructor(public running: number, public queued: number,  public remaining: number,  public error: number) {}
+    constructor(public running: number, public remaining: number, public error: number) {
+    }
 }

vripper-ui/src/app/status-bar/status-bar.component.html

@@ -4,7 +4,6 @@
   <span *ngIf="downloadSpeed$ | async as speed">{{ speed.speed + '/s' }}</span>
   <ng-container *ngIf="globalState$ | async as state">
     <span>Downloading: {{ state!.running }}</span>
-    <span>Pending: {{ state!.queued }}</span>
     <span>Remaining: {{ state!.remaining }}</span>
     <span>Error: {{ state!.error }}</span>
   </ng-container>

vripper-ui/src/app/status-bar/status-bar.component.ts

@@ -16,7 +16,7 @@ export class StatusBarComponent implements OnInit, OnDestroy {
   }
 
   downloadSpeed$: Subject<DownloadSpeed> = new BehaviorSubject(new DownloadSpeed('0 B'));
-  globalState$: Subject<GlobalState> = new BehaviorSubject(new GlobalState(0, 0, 0, 0));
+  globalState$: Subject<GlobalState> = new BehaviorSubject(new GlobalState(0, 0, 0));
   selected$: Subject<number> = new BehaviorSubject(0);
   subscriptions: Subscription[] = [];
 

vripper-ui/src/environments/environment.prod.ts

@@ -2,5 +2,5 @@ export const environment = {
   production: true,
   localhost: `${window.location.protocol}//${window.location.host}`,
   ws: `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}`,
-  version: '3.0.0'
+  version: '3.0.1'
 };

vripper-ui/src/environments/environment.ts

@@ -6,7 +6,7 @@ export const environment = {
   production: false,
   localhost: 'http://localhost:8080',
   ws: 'ws://localhost:8080',
-  version: '3.0.0'
+  version: '3.0.1'
 };
 
 /*