v2.10.5

Add system notification
Bug fixes

CHANGELOG.md

@@ -1,5 +1,58 @@
 # Changelog
 
+## [2.10.5] - 2020-01-18
+### Changed
+- Bug fixes
+
+## [2.10.4] - 2020-01-18
+### Changed
+- Zero value for total max download will disable the limit
+- Add system notification
+- Bug fixes
+
+## [2.10.3] - 2020-01-16
+### Changed
+- Some bug fixes
+- Enhance download queue logic
+- Clear cache button
+- Fix the partial status
+- Sanitize URLs with spaces
+### Added
+- Support for Postimg host
+- Support for Imagevenue host
+
+## [2.10.2] - 2019-12-29
+### Changed
+- Upgrade electron to v7
+- Bug fix
+
+## [2.10.1] - 2019-12-27
+### Changed
+- Bug fix
+
+## [2.10.0] - 2019-12-27
+### Changed
+- Fix spring dependency bug
+### Added
+- Add support for imgspice
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Fix build issue
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Decrease timeout for fast fail
+### Added
+- Allow many hosts to run at the same time
+- Add title to gallery
+- Add global concurrent downloads
+- Add counter badge in link collector
+
+## [2.8.1] - 2019-12-24
+### Changed
+- Fix issue with path creation
+
 ## [2.8.0] - 2019-12-24
 ### Added
 - Photo gallery feature

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.8.0</version>
+    <version>2.10.5</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/main.js

@@ -23,6 +23,11 @@ process.on("uncaughtException", err => {
 });
 
 function createWindow() {
+
+  if (process.platform === 'win32') {
+    app.setAppUserModelId("tn.mnlr.vripper");
+  }
+
   let icon;
   if(process.platform === "win32") {
     icon = __dirname + '/icon.ico';
@@ -59,51 +64,51 @@ function createWindow() {
   });
 }
 
-getPort().then(port => {
-  serverPort = port;
-  ipcMain.on("get-port", event => {
-    event.reply("port", port);
-  });
-  let javaBinPath;
-  if(appDir !== undefined) {
-    javaBinPath = path.join(appDir, "java-runtime/bin/java");
-  } else {
-    if(process.platform === 'darwin') {
-      javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
-    } else {
-      javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
-    }
-  }
-  let jarPath;
-  if(appDir !== undefined) {
-    jarPath = path.join(appDir, "bin/vripper-server.jar");
-  } else {
-    if(process.platform === 'darwin') {
-      jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
-    } else {
-      jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
-    }
-  }
-  vripperServer = spawn(javaBinPath, [
-    "-Xms256m",
-    "-Dvripper.server.port=" + port,
-    "-jar",
-    jarPath
-  ], {
-    stdio: 'ignore'
-  });
-  vripperServer.on('exit', (code, signal) => {
-    console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
-    terminated = true;
-    app.quit();
-  });
-});
-
 const gotTheLock = app.requestSingleInstanceLock();
 
 if (!gotTheLock) {
   app.quit();
 } else {
+  getPort().then(port => {
+    serverPort = port;
+    ipcMain.on("get-port", event => {
+      event.reply("port", port);
+    });
+    let javaBinPath;
+    if(appDir !== undefined) {
+      javaBinPath = path.join(appDir, "java-runtime/bin/java");
+    } else {
+      if(process.platform === 'darwin') {
+        javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+      } else {
+        javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+      }
+    }
+    let jarPath;
+    if(appDir !== undefined) {
+      jarPath = path.join(appDir, "bin/vripper-server.jar");
+    } else {
+      if(process.platform === 'darwin') {
+        jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+      } else {
+        jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+      }
+    }
+    vripperServer = spawn(javaBinPath, [
+      "-Xms256m",
+      "-Dvripper.server.port=" + port,
+      "-jar",
+      jarPath
+    ], {
+      stdio: 'ignore'
+    });
+    vripperServer.on('exit', (code, signal) => {
+      console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
+      terminated = true;
+      app.quit();
+    });
+  });
+  
   app.on("second-instance", (event, commandLine, workingDirectory) => {
     if (win) {
       if (win.isMinimized()) win.restore();

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.8.0",
+  "version": "2.10.5",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -70,8 +70,8 @@
     "dist": "node pre-build.js && electron-builder"
   },
   "devDependencies": {
-    "electron": "^6.0.9",
-    "electron-builder": "^21.2.0"
+    "electron": "^7.1.9",
+    "electron-builder": "^22.2.0"
   },
   "dependencies": {
     "axios": "^0.19.0",

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.8.0</version>
+    <version>2.10.5</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>2.8.0</version>
+        <version>2.10.5</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>

vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java

@@ -0,0 +1,24 @@
+package tn.mnlr.vripper;
+
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.event.ContextRefreshedEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+import tn.mnlr.vripper.services.PersistenceService;
+
+@Component
+public class EventListenerBean {
+
+    @Autowired
+    private PersistenceService persistenceService;
+
+    @Getter
+    private static boolean init = false;
+
+    @EventListener
+    public void onApplicationEvent(ContextRefreshedEvent event) {
+        init = true;
+        persistenceService.restore();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/SpringContext.java

@@ -27,7 +27,9 @@ public class SpringContext implements ApplicationContextAware {
 
     public static void close() {
         logger.info("Application terminating...");
-        context.close();
+        if (context != null) {
+            context.close();
+        }
 
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java

@@ -1,21 +1,10 @@
 package tn.mnlr.vripper;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.stereotype.Component;
-import tn.mnlr.vripper.exception.VripperException;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.PersistenceService;
-import tn.mnlr.vripper.services.VipergirlsAuthService;
 
-import javax.annotation.PostConstruct;
-import java.io.File;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -27,51 +16,11 @@ public class VripperApplication {
 
     public static void main(String[] args) {
         try {
+            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
             SpringApplication.run(VripperApplication.class, args);
         } catch (Exception e) {
             logger.error("Failed to run the application", e);
         }
     }
-
-    @Component
-    public class AppCommandRunner implements CommandLineRunner {
-
-        @Autowired
-        private VipergirlsAuthService authService;
-
-        @Autowired
-        private PersistenceService persistenceService;
-
-        @Autowired
-        private AppSettingsService appSettingsService;
-
-        @Value("${base.dir}")
-        private String baseDir;
-
-        @Getter
-        private String dataPath;
-
-        @PostConstruct
-        public void init() {
-            dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
-        }
-
-        @Override
-        public void run(String... args) {
-
-            persistenceService.restore();
-            appSettingsService.restore();
-            registerShutdownHook();
-
-            try {
-                authService.authenticate();
-            } catch (VripperException e) {
-                logger.error("Cannot authenticate user with ViperGirls", e);
-            }
-        }
-
-        private void registerShutdownHook() {
-            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
-        }
-    }
 }
+

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -32,6 +32,7 @@ import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.util.Iterator;
+import java.util.Objects;
 
 @Service
 abstract public class Host {
@@ -65,6 +66,11 @@ abstract public class Host {
 
     abstract public String getLookup();
 
+    @Override
+    public int hashCode() {
+        return Objects.hash(pathService);
+    }
+
     public boolean isSupported(String url) {
         return url.contains(getLookup());
     }
@@ -98,11 +104,10 @@ abstract public class Host {
             imageFileData.setImageName(formatImageFileName);
             File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
             logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
+            logger.debug(String.format("Creating %s", destinationFolder.getPath()));
+            destinationFolder.mkdirs();
             if (!destinationFolder.exists()) {
-                logger.debug(String.format("Creating %s", destinationFolder.getPath()));
-                if (destinationFolder.mkdirs()) {
-                    logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
-                }
+                logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
             }
 
             HttpClient client = cm.getClient().build();
@@ -115,7 +120,7 @@ abstract public class Host {
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
-                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
+                File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
                 try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
                     image.setTotal(response.getEntity().getContentLength());
                     logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
@@ -234,8 +239,17 @@ abstract public class Host {
 
     protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Host host = (Host) o;
+        return Objects.equals(getHost(), host.getHost());
+    }
+
     @Getter
     public static class Response {
+
         protected Response(Document document, Header[] headers) {
             this.document = document;
             this.headers = headers;

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageVenueHost.java

@@ -0,0 +1,79 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.net.URI;
+
+@Service
+public class ImageVenueHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImageVenueHost.class);
+
+    private static final String host = "imagevenue.com";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@id='thepic']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        //Sadly, they do not support https.
+        //If they add such support in the future,
+        //then we should automatically adapt the URL here as done elsewhere.
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            if(xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
+                //Button detected. No need to actually click it, just make the call again.
+                resp = getResponse(url, context);
+                doc = resp.getDocument();
+            }
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        if (imgNode == null) {
+            throw new HostException("Failed to locate image");
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            URI baseUri = new URI(url);
+            imageFileData.setImageUrl(new URI(baseUri.getScheme(), baseUri.getHost(), '/'+imgUrl, null).toString());
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch(Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgSpiceHost.java

@@ -0,0 +1,62 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class ImgSpiceHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImgSpiceHost.class);
+
+    private static final String host = "imgspice.com";
+    private static final String IMG_XPATH = "//img[@id='imgpreview']";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PostImgHost.java

@@ -0,0 +1,61 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.util.Optional;
+
+@Service
+public class PostImgHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PostImgHost.class);
+
+    private static final String host = "postimg.cc";
+    private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
+    private static final String IMG_XPATH = "//a[@id='download']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Document doc = getResponse(url, context).getDocument();
+
+        Node urlNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            urlNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> getDefaultImageName(url));
+
+            imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
+            imageFileData.setImageName(imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java

@@ -1,156 +1,83 @@
 package tn.mnlr.vripper.q;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
 
-import java.util.Arrays;
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
+import java.util.Map;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingDeque;
 
 @Service
 public class DownloadQ {
 
-    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
-
     private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
 
     @Autowired
     private AppStateService appStateService;
 
+    private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> downloadQ = new ConcurrentHashMap<>();
     @Autowired
-    private ExecutionService executionService;
+    private AppSettingsService appSettingsService;
+    @Autowired
+    private List<Host> hosts;
 
-    private BlockingQueue<DownloadJob> downloadQ = new LinkedBlockingQueue<>();
-
-    @Getter
-    private boolean notPauseQ = true;
+    @PostConstruct
+    private void init() {
+        hosts.forEach(host -> downloadQ.put(host, new LinkedBlockingDeque<>()));
+    }
 
     public void put(Image image) throws InterruptedException {
-        synchronized (appStateService) {
-            logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
-            image.init();
-            DownloadJob downloadJob = new DownloadJob(image);
-            downloadQ.put(downloadJob);
-            appStateService.newDownloadJob(downloadJob);
+        logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
+        image.init();
+        DownloadJob downloadJob = new DownloadJob(image);
+        downloadQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
+        appStateService.newDownloadJob(downloadJob);
+    }
+
+    public void remove(final DownloadJob downloadJob) {
+        downloadQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
+    }
+
+    public List<DownloadJob> peek() {
+        List<DownloadJob> downloadJobs = new ArrayList<>();
+        if (hosts.size() == 0) {
+            return downloadJobs;
         }
+        for (Host host : hosts) {
+            Iterator<DownloadJob> it = downloadQ.get(host).iterator();
+            for (int i = 0; i < appSettingsService.getMaxThreads(); i++) {
+                DownloadJob downloadJob = it.hasNext() ? it.next() : null;
+                if (downloadJob != null) {
+                    downloadJobs.add(downloadJob);
+                }
+            }
+        }
+        return downloadJobs;
     }
 
-    DownloadJob take() throws InterruptedException {
-        DownloadJob downloadJob = downloadQ.take();
-        logger.debug(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
-        return downloadJob;
-    }
-
-    public synchronized void enqueue(Post post) throws InterruptedException {
+    public void enqueue(Post post) throws InterruptedException {
         for (Image image : post.getImages()) {
             put(image);
         }
     }
 
-    public void restart(String postId) throws InterruptedException {
-        synchronized (appStateService) {
-            if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
-                logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
-                return;
-            }
-            List<Image> images = appStateService.getPost(postId)
-                    .getImages()
-                    .stream()
-                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                    .collect(Collectors.toList());
-            if (images.isEmpty()) {
-                return;
-            }
-            appStateService.getPost(postId).setStatus(Post.Status.PENDING);
-            logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
-            for (Image image : images) {
-                put(image);
-            }
-        }
-    }
-
-    private void removeScheduled(Image image) {
-        synchronized (appStateService) {
-            image.setStatus(Image.Status.STOPPED);
-            logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
-
-            Iterator<DownloadJob> iterator = downloadQ.iterator();
-            boolean removed = false;
-            while (iterator.hasNext()) {
-                DownloadJob next = iterator.next();
-                if (next.getImage().getPostId().equals(image.getPostId())) {
-                    iterator.remove();
-                    appStateService.doneDownloadJob(image);
-                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
-                    removed = true;
-                    break;
-                }
-            }
-
-            if (!removed) {
-                logger.debug(String.format("Job for %s does not exist", image.getUrl()));
-            }
-
-            image.cleanup();
-        }
-    }
-
-    private void removeRunning(String postId) {
-        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
-        executionService.stop(postId);
-    }
-
-
-    public void stop(String postId) {
-        try {
-            synchronized (appStateService) {
-                if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
-                    return;
-                }
-                notPauseQ = false;
-                appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
-                List<Image> images = appStateService.getPost(postId)
-                        .getImages()
-                        .stream()
-                        .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                        .collect(Collectors.toList());
-                if (images.isEmpty()) {
-                    return;
-                }
-                logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
-                images.forEach(this::removeScheduled);
-                removeRunning(postId);
-            }
-        } finally {
-            notPauseQ = true;
-        }
-    }
-
     public int size() {
-        return downloadQ.size();
+        return downloadQ.values().stream().mapToInt(BlockingDeque::size).sum();
     }
 
-    public void stopAll() {
-        synchronized (appStateService) {
-            appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
-        }
-    }
-
-    public void restartAll() throws InterruptedException {
-        synchronized (appStateService) {
-            for (Post post : appStateService.getCurrentPosts().values()) {
-                String postId = post.getPostId();
-                restart(postId);
-            }
-        }
+    public Iterable<? extends Map.Entry<Host, BlockingDeque<DownloadJob>>> entries() {
+        return downloadQ.entrySet();
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -9,17 +9,17 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
 import tn.mnlr.vripper.services.ThumbnailGenerator;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
+import java.time.Duration;
 import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -29,6 +29,8 @@ public class ExecutionService {
 
     private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
 
+    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
+
     @Autowired
     private DownloadQ downloadQ;
 
@@ -43,30 +45,35 @@ public class ExecutionService {
 
     @Autowired
     private AppSettingsService appSettingsService;
-
-    private final AtomicInteger threadCount = new AtomicInteger();
-
-    private ExecutorService executor = Executors.newFixedThreadPool(10);
+    private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
+    private boolean notPauseQ = true;
+    private ExecutorService executor = Executors.newFixedThreadPool(20);
 
     private Thread executionThread;
 
+    BlockingQueue<DownloadJob> queue = new LinkedBlockingQueue<>();
+
     private RetryPolicy<Object> retryPolicy;
 
     private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
 
     private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
+    private Thread pollThread;
 
     @PostConstruct
     private void init() {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withBackoff(10, 60, ChronoUnit.SECONDS)
-                .withMaxRetries(4)
+                .withBackoff(1, 10, ChronoUnit.SECONDS)
+                .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+                .withMaxAttempts(3)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 
         executionThread = new Thread(this::start, "Executor thread");
+        pollThread = new Thread(this::poll, "Polling thread");
+        pollThread.start();
         executionThread.start();
     }
 
@@ -77,12 +84,12 @@ public class ExecutionService {
         executor.shutdown();
         appStateService.getCurrentPosts().keySet().forEach(p -> {
             logger.debug(String.format("Stopping download jobs for %s", p));
-            this.stop(p);
+            this.stopRunning(p);
         });
         executor.awaitTermination(10, TimeUnit.SECONDS);
     }
 
-    public void stop(String postId) {
+    public void stopRunning(String postId) {
         List<DownloadJob> data = running
                 .stream()
                 .filter(e -> e.getImage().getPostId().equals(postId))
@@ -92,81 +99,182 @@ public class ExecutionService {
 
         data.forEach(e -> {
             futures.get(e.getImage().getUrl()).cancel(true);
-            if(e.getImageFileData().getImageRequest() != null) {
+            if (e.getImageFileData().getImageRequest() != null) {
                 e.getImageFileData().getImageRequest().abort();
             }
             e.getImage().cleanup();
         });
     }
 
-    private boolean canRun() {
-        boolean canRun = threadCount.get() < settings.getMaxThreads();
-        if (canRun && downloadQ.isNotPauseQ()) {
-            threadCount.incrementAndGet();
+
+    public synchronized void stopAll() {
+        appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
+    }
+
+    public synchronized void restartAll() throws InterruptedException {
+        for (Post post : appStateService.getCurrentPosts().values()) {
+            String postId = post.getPostId();
+            restart(postId);
+        }
+    }
+
+    public synchronized void restart(String postId) throws InterruptedException {
+        if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
+            logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
+            return;
+        }
+        List<Image> images = appStateService.getPost(postId)
+                .getImages()
+                .stream()
+                .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                .collect(Collectors.toList());
+        if (images.isEmpty()) {
+            return;
+        }
+        appStateService.getPost(postId).setStatus(Post.Status.PENDING);
+        logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
+        for (Image image : images) {
+            downloadQ.put(image);
+        }
+    }
+
+    private synchronized void removeScheduled(Image image) {
+        image.setStatus(Image.Status.STOPPED);
+        logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
+
+        boolean removed = false;
+        main:
+        for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : downloadQ.entries()) {
+            Iterator<DownloadJob> iterator = entry.getValue().iterator();
+            while (iterator.hasNext()) {
+                DownloadJob next = iterator.next();
+                if (next.getImage().getPostId().equals(image.getPostId())) {
+                    iterator.remove();
+                    appStateService.doneDownloadJob(image);
+                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
+                    removed = true;
+                    break main;
+                }
+            }
+        }
+
+        if (!removed) {
+            logger.debug(String.format("Job for %s does not exist", image.getUrl()));
+        }
+
+        image.cleanup();
+    }
+
+    private synchronized void removeRunning(String postId) {
+        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
+        stopRunning(postId);
+    }
+
+    public synchronized void stop(String postId) {
+        try {
+            if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
+                return;
+            }
+            notPauseQ = false;
+            appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
+            List<Image> images = appStateService.getPost(postId)
+                    .getImages()
+                    .stream()
+                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                    .collect(Collectors.toList());
+            if (images.isEmpty()) {
+                return;
+            }
+            logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
+            images.forEach(this::removeScheduled);
+            removeRunning(postId);
+        } finally {
+            notPauseQ = true;
+        }
+    }
+
+    private boolean canRun(Host host) {
+        boolean canRun;
+        AtomicInteger count = threadCount.get(host);
+        if (count == null) {
+            threadCount.put(host, new AtomicInteger(0));
+        }
+        canRun = threadCount.get(host).get() < settings.getMaxThreads() && (settings.getMaxTotalThreads() == 0 || threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getMaxTotalThreads());
+        if (canRun && notPauseQ) {
+            threadCount.get(host).incrementAndGet();
             return true;
         }
         return false;
     }
 
-    public void start() {
+    public void poll() {
         while (!Thread.interrupted()) {
-            if (canRun()) {
-                DownloadJob take;
-                try {
-                    take = downloadQ.take();
-                    if (take == null) {
-                        continue;
+            try {
+                List<DownloadJob> peek = downloadQ.peek();
+                for (DownloadJob downloadJob : peek) {
+                    if (canRun(downloadJob.getImage().getHost())) {
+                        queue.offer(downloadJob);
+                        downloadQ.remove(downloadJob);
                     }
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    break;
                 }
-
-                DownloadJob finalTake = take;
-                Runnable task = () -> {
-                    running.add(finalTake);
-
-                    Failsafe.with(retryPolicy)
-                            .onFailure(e -> {
-                                if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
-                                    logger.debug("Job successfully interrupted");
-                                    return;
-                                }
-                                logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
-                                finalTake.getImage().setStatus(Image.Status.ERROR);
-                            })
-                            .onComplete(e -> {
-                                appStateService.doneDownloadJob(finalTake.getImage());
-                                logger.debug(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
-                                if (appSettingsService.isViewPhotos()) {
-                                    VripperApplication.commonExecutor.submit(
-                                            () -> thumbnailGenerator.getThumbnails()
-                                                    .get(new ThumbnailGenerator.CacheKey(finalTake.getImage().getPostId(), finalTake.getImageFileData().getFileName())));
-                                }
-                                synchronized (threadCount) {
-                                    threadCount.decrementAndGet();
-                                    running.remove(finalTake);
-                                    futures.remove(finalTake.getImage().getUrl());
-                                    threadCount.notify();
-                                }
-                            })
-                            .get(finalTake::call);
-                };
-                logger.debug(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
-                futures.put(finalTake.getImage().getUrl(), executor.submit(task));
-            } else {
                 synchronized (threadCount) {
-                    try {
-                        threadCount.wait(2_000);
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                        break;
-                    }
+                    threadCount.wait(2_000);
                 }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
             }
         }
     }
 
+    public void start() {
+        while (!Thread.interrupted()) {
+            try {
+                push(queue.take());
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            } catch (Exception e) {
+                logger.error("Execution Service failed", e);
+                break;
+            }
+        }
+    }
+
+    private void push(DownloadJob take) {
+        Runnable task = () -> {
+            running.add(take);
+
+            Failsafe.with(retryPolicy)
+                    .onFailure(e -> {
+                        if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
+                            logger.debug("Job successfully interrupted");
+                            return;
+                        }
+                        logger.error(String.format("Failed to download %s after %d tries", take.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
+                        take.getImage().setStatus(Image.Status.ERROR);
+                    })
+                    .onComplete(e -> {
+                        appStateService.doneDownloadJob(take.getImage());
+                        logger.debug(String.format("Finished downloading %s", take.getImage().getUrl()));
+                        if (appSettingsService.isViewPhotos()) {
+                            VripperApplication.commonExecutor.submit(
+                                    () -> thumbnailGenerator.getThumbnails()
+                                            .get(new ThumbnailGenerator.CacheKey(take.getImage().getPostId(), take.getImageFileData().getFileName())));
+                        }
+                        threadCount.get(take.getImage().getHost()).decrementAndGet();
+                        running.remove(take);
+                        futures.remove(take.getImage().getUrl());
+                        synchronized (threadCount) {
+                            threadCount.notify();
+                        }
+                    })
+                    .get(take::call);
+        };
+        logger.debug(String.format("Scheduling a job for %s", take.getImage().getUrl()));
+        futures.put(take.getImage().getUrl(), executor.submit(task));
+    }
+
     public int runningCount() {
         return running.size();
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -11,6 +11,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.exception.ValidationException;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
@@ -24,6 +25,8 @@ import java.util.prefs.Preferences;
 @Setter
 public class AppSettingsService {
 
+    private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
+
     @Value("${base.dir}")
     private String defaultDownloadPath;
 
@@ -33,6 +36,7 @@ public class AppSettingsService {
 
     private final String DOWNLOAD_PATH = "DOWNLOAD_PATH";
     private final String MAX_THREADS = "MAX_THREADS";
+    private int maxTotalThreads;
     private final String AUTO_START = "AUTO_START";
     private final String V_LOGIN = "VLOGIN";
     private final String V_USERNAME = "VUSERNAME";
@@ -45,6 +49,12 @@ public class AppSettingsService {
     private final String CLEAR = "CLEAR";
     private final String DARK_THEME = "DARK_THEME";
     private final String VIEW_PHOTOS = "VIEW_PHOTOS";
+    private final String NOTIFICATION = "NOTIFICATION";
+
+    @PostConstruct
+    private void init() {
+        restore();
+    }
 
     private String downloadPath;
     private int maxThreads;
@@ -60,6 +70,7 @@ public class AppSettingsService {
     private boolean clearCompleted;
     private boolean darkTheme;
     private boolean viewPhotos;
+    private boolean notificationEnabled;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -73,6 +84,7 @@ public class AppSettingsService {
 
         downloadPath = prefs.get(DOWNLOAD_PATH, defaultDownloadPath);
         maxThreads = prefs.getInt(MAX_THREADS, 4);
+        maxTotalThreads = prefs.getInt(MAX_TOTAL_THREADS, 0);
         autoStart = prefs.getBoolean(AUTO_START, true);
         vLogin = prefs.getBoolean(V_LOGIN, false);
         vUsername = prefs.get(V_USERNAME, "");
@@ -85,6 +97,7 @@ public class AppSettingsService {
         clearCompleted = prefs.getBoolean(CLEAR, false);
         darkTheme = prefs.getBoolean(DARK_THEME, false);
         viewPhotos = prefs.getBoolean(VIEW_PHOTOS, false);
+        notificationEnabled = prefs.getBoolean(NOTIFICATION, false);
     }
 
     @PreDestroy
@@ -92,6 +105,7 @@ public class AppSettingsService {
 
         prefs.put(DOWNLOAD_PATH, downloadPath);
         prefs.putInt(MAX_THREADS, maxThreads);
+        prefs.putInt(MAX_TOTAL_THREADS, maxTotalThreads);
         prefs.putBoolean(AUTO_START, autoStart);
         prefs.putBoolean(V_LOGIN, vLogin);
         prefs.put(V_USERNAME, vUsername);
@@ -104,6 +118,7 @@ public class AppSettingsService {
         prefs.putBoolean(CLEAR, clearCompleted);
         prefs.putBoolean(DARK_THEME, darkTheme);
         prefs.putBoolean(VIEW_PHOTOS, viewPhotos);
+        prefs.putBoolean(NOTIFICATION, notificationEnabled);
 
         try {
             prefs.sync();
@@ -126,6 +141,10 @@ public class AppSettingsService {
             throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
         }
 
+        if (settings.getMaxTotalThreads() < 0) {
+            throw new ValidationException(String.format("Invalid max global concurrent download settings, values must be in greater than %d", 1));
+        }
+
         if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
             throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
         }
@@ -159,6 +178,8 @@ public class AppSettingsService {
         private String downloadPath;
         @JsonProperty("maxThreads")
         private int maxThreads;
+        @JsonProperty("maxTotalThreads")
+        private int maxTotalThreads;
         @JsonProperty("autoStart")
         private boolean autoStart;
         @JsonProperty("vLogin")
@@ -181,10 +202,13 @@ public class AppSettingsService {
         private boolean clearCompleted;
         @JsonProperty("viewPhotos")
         private boolean viewPhotos;
+        @JsonProperty("notification")
+        private boolean notification;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted, boolean viewPhotos) {
+        public Settings(String downloadPath, int maxThreads, int maxTotalThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted, boolean viewPhotos, boolean notification) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
+            this.maxTotalThreads = maxTotalThreads;
             this.autoStart = autoStart;
             this.vLogin = vLogin;
             this.vUsername = vUsername;
@@ -196,6 +220,7 @@ public class AppSettingsService {
             this.threadSubLocation = threadSubLocation;
             this.clearCompleted = clearCompleted;
             this.viewPhotos = viewPhotos;
+            this.notification = notification;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -59,7 +59,7 @@ public class AppStateService {
         return currentPosts.get(postId);
     }
 
-    public synchronized void newDownloadJob(DownloadJob downloadJob) {
+    public void newDownloadJob(DownloadJob downloadJob) {
         String postId = downloadJob.getImage().getPostId();
         checkKeyRunningPosts(postId);
         runningPosts.get(postId).incrementAndGet();
@@ -67,7 +67,7 @@ public class AppStateService {
 
     public synchronized void postDownloadingUpdate(String postId) {
         Post post = currentPosts.get(postId);
-        if (!post.getStatus().equals(Post.Status.DOWNLOADING)) {
+        if (!post.getStatus().equals(Post.Status.DOWNLOADING) && !post.getStatus().equals(Post.Status.PARTIAL)) {
             post.setStatus(Post.Status.DOWNLOADING);
             livePostsState.onNext(post);
         }

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -26,9 +26,9 @@ public class ConnectionManager {
     private PoolingHttpClientConnectionManager pcm;
 
     private RequestConfig rc = RequestConfig.custom()
-            .setConnectionRequestTimeout(10_000)
-            .setConnectTimeout(10_000)
-            .setSocketTimeout(10_000)
+            .setConnectionRequestTimeout(5_000)
+            .setConnectTimeout(5_000)
+            .setSocketTimeout(5_000)
             .setCookieSpec(CookieSpecs.STANDARD)
             .build();
 
@@ -54,13 +54,13 @@ public class ConnectionManager {
     }
 
     public HttpGet buildHttpGet(String url) {
-        HttpGet httpGet = new HttpGet(url);
+        HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
         httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpGet;
     }
 
     public HttpPost buildHttpPost(String url) {
-        HttpPost httpPost = new HttpPost(url);
+        HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
         httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpPost;
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeed.java

@@ -13,7 +13,7 @@ public class DownloadSpeed {
     }
 
     private String formatSI(long bytes) {
-        return humanReadableByteCount(bytes, true);
+        return humanReadableByteCount(bytes, false);
     }
 
     private String humanReadableByteCount(long bytes, boolean si) {

vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java

@@ -7,9 +7,10 @@ import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.EventListenerBean;
 import tn.mnlr.vripper.SpringContext;
-import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
@@ -38,8 +39,11 @@ public class PersistenceService {
     @Autowired
     private AppStateService stateService;
 
-    @Autowired
-    private VripperApplication.AppCommandRunner appCommandRunner;
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Getter
+    private String dataPath;
 
     private ObjectMapper om;
 
@@ -61,7 +65,8 @@ public class PersistenceService {
 
     @PostConstruct
     public void init() {
-        File dataFile = new File(appCommandRunner.getDataPath());
+        dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
+        File dataFile = new File(dataPath);
         if (!dataFile.exists()) {
             try {
                 if (dataFile.getParentFile().mkdirs()) {
@@ -92,12 +97,14 @@ public class PersistenceService {
         this.subscription.dispose();
         logger.info(String.format("Destroying %s", PersistenceService.class.getSimpleName()));
         logger.info("Persisting data before destroying");
-        this.persist(stateService.getCurrentPosts());
+        if (EventListenerBean.isInit()) {
+            this.persist(stateService.getCurrentPosts());
+        }
     }
 
     private void persist(Map<String, Post> currentPosts) {
 
-        try (PrintWriter out = new PrintWriter(appCommandRunner.getDataPath(), StandardCharsets.UTF_8)) {
+        try (PrintWriter out = new PrintWriter(dataPath, StandardCharsets.UTF_8)) {
             out.print(om.writeValueAsString(currentPosts));
         } catch (IOException e) {
             logger.error("Failed to persist app state", e);
@@ -111,11 +118,11 @@ public class PersistenceService {
         } catch (IOException e) {
             logger.error("Failed to read app state", e);
             long timestamp = new Date().getTime();
-            logger.warn(String.format("trying to rename old data file from %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + "." + timestamp + ".old"));
+            logger.warn(String.format("trying to rename old data file from %s to %s", dataPath, dataPath + "." + timestamp + ".old"));
             try {
-                Files.move(new File(appCommandRunner.getDataPath()).toPath(), new File(appCommandRunner.getDataPath() + "." + timestamp + ".old").toPath());
+                Files.move(new File(dataPath).toPath(), new File(dataPath + "." + timestamp + ".old").toPath());
             } catch (IOException ex) {
-                logger.error(String.format("Failed to rename %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + ".old"));
+                logger.error(String.format("Failed to rename %s to %s", dataPath, dataPath + ".old"));
                 SpringContext.close();
             }
         }
@@ -127,7 +134,7 @@ public class PersistenceService {
 
         String jsonContent = null;
         try {
-            jsonContent = String.join("", Files.readAllLines(Paths.get(appCommandRunner.getDataPath()), StandardCharsets.UTF_8));
+            jsonContent = String.join("", Files.readAllLines(Paths.get(dataPath), StandardCharsets.UTF_8));
         } catch (Exception e) {
             logger.error("data file cannot be read, previous state cannot be restored", e);
             SpringContext.close();

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -45,7 +45,7 @@ public class PostParser {
 
     }
 
-    public void addPost(String postId, String threadId) throws PostParseException {
+    public synchronized void addPost(String postId, String threadId) throws PostParseException {
 
         if (appStateService.getCurrentPosts().containsKey(postId)) {
             logger.warn(String.format("skipping %s, already loaded", postId));

vripper-server/src/main/java/tn/mnlr/vripper/services/ThumbnailGenerator.java

@@ -10,23 +10,35 @@ import org.imgscalr.Scalr;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.springframework.util.FileSystemUtils;
 
 import javax.annotation.PostConstruct;
 import javax.imageio.ImageIO;
 import java.awt.image.BufferedImage;
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
 
 @Service
 public class ThumbnailGenerator {
 
     @Getter
     private LoadingCache<CacheKey, byte[]> thumbnails;
+
     @Value("${base.dir}")
     private String baseDir;
+
     @Autowired
     private PathService pathService;
+
+    @Getter
     private File cacheFolder;
     CacheLoader<CacheKey, byte[]> loader = new CacheLoader<>() {
         @Override
@@ -42,25 +54,69 @@ public class ThumbnailGenerator {
     @PostConstruct
     private void init() throws Exception {
         cacheFolder = new File(baseDir + File.separator + ".vripper" + File.separator + "cache");
+        cacheFolder.mkdirs();
         if (!cacheFolder.exists()) {
-            if (!cacheFolder.mkdirs()) {
-                throw new Exception(String.format("%s could not be created", cacheFolder.toString()));
-            }
+            throw new Exception(String.format("%s could not be created", cacheFolder.toString()));
         }
         thumbnails = CacheBuilder.newBuilder()
                 .maximumSize(20000)
                 .build(loader);
     }
 
+    public void clearCache() {
+        thumbnails.invalidateAll();
+        for (File file : Optional.ofNullable(cacheFolder.listFiles()).orElse(new File[]{})) {
+            FileSystemUtils.deleteRecursively(file);
+        }
+    }
+
+    public long cacheSize() {
+
+        final AtomicLong size = new AtomicLong(0);
+
+        try {
+            Files.walkFileTree(cacheFolder.toPath(), new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+
+                    System.out.println("skipped: " + file + " (" + exc + ")");
+                    // Skip folders that can't be traversed
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+
+                    if (exc != null)
+                        System.out.println("had trouble traversing: " + dir + " (" + exc + ")");
+                    else
+                        size.addAndGet(dir.toFile().length());
+                    // Ignore errors traversing a folder
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new AssertionError("walkFileTree will not throw IOException if the FileVisitor does not");
+        }
+
+        return size.get();
+    }
+
     private File generateThumbnail(File inputFile, String postId) throws Exception {
         if (!inputFile.exists()) {
             throw new Exception(String.format("Input file %s does not exist", inputFile.toString()));
         }
         File postsCacheFolder = new File(cacheFolder, postId);
+        postsCacheFolder.mkdirs();
         if (!postsCacheFolder.exists()) {
-            if (!postsCacheFolder.mkdirs()) {
-                throw new Exception(String.format("%s could not be created", postsCacheFolder.toString()));
-            }
+            throw new Exception(String.format("%s could not be created", postsCacheFolder.toString()));
         }
 
         File thumbFile = new File(postsCacheFolder, inputFile.getName());

vripper-server/src/main/java/tn/mnlr/vripper/services/VGHandler.java

@@ -57,21 +57,18 @@ public class VGHandler {
             if (queuedVGLink.getPostId() != null) {
                 postParser.addPost(queuedVGLink.getPostId(), queuedVGLink.getThreadId());
             } else {
-                appStateService.getGrabQueue().put(queuedVGLink.getLink(), queuedVGLink);
-                appStateService.getLiveGrabQueue().onNext(queuedVGLink);
-
                 Callable<Void> cl = () -> {
                     List<VRPostState> vrPostStates = cache.get(queuedVGLink.getThreadId());
                     logger.debug(String.format("%d found for %s", vrPostStates.size(), queuedVGLink.getLink()));
                     if (vrPostStates.size() == 1) {
                         postParser.addPost(vrPostStates.get(0).getPostId(), vrPostStates.get(0).getThreadId());
-                        remove(queuedVGLink.getLink());
                         logger.debug(String.format("threadId %s, postId %s is added automatically for download", queuedVGLink.getThreadId(), queuedVGLink.getPostId()));
+                    } else {
+                        appStateService.getGrabQueue().put(queuedVGLink.getLink(), queuedVGLink);
+                        appStateService.getLiveGrabQueue().onNext(queuedVGLink);
                     }
-
                     return null;
                 };
-
                 VripperApplication.commonExecutor.submit(cl);
             }
         }

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostParser.java

@@ -22,6 +22,7 @@ import javax.xml.parsers.SAXParserFactory;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -39,8 +40,9 @@ class VRPostParser {
 
     private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
             .handleIf(e -> e instanceof IOException)
-            .withBackoff(5, 30, ChronoUnit.SECONDS)
-            .withMaxRetries(4)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
             .abortOn(InterruptedException.class)
             .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 

vripper-server/src/main/java/tn/mnlr/vripper/services/VRThreadParser.java

@@ -21,6 +21,7 @@ import javax.xml.parsers.SAXParserFactory;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -35,8 +36,9 @@ public class VRThreadParser {
 
     private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
             .handleIf(e -> e instanceof IOException)
-            .withBackoff(5, 30, ChronoUnit.SECONDS)
-            .withMaxRetries(4)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
             .abortOn(InterruptedException.class)
             .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
     private static SAXParserFactory factory = SAXParserFactory.newInstance();

vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java

@@ -59,6 +59,11 @@ public class VipergirlsAuthService {
     @PostConstruct
     private void init() {
         context.setCookieStore(new BasicCookieStore());
+        try {
+            authenticate();
+        } catch (VripperException e) {
+            logger.error("Cannot authenticate user with ViperGirls", e);
+        }
     }
 
     @PreDestroy

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/GalleryEndpoint.java

@@ -1,5 +1,8 @@
 package tn.mnlr.vripper.web.restendpoints;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.NonNull;
@@ -26,6 +29,7 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.*;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -89,6 +93,7 @@ public class GalleryEndpoint {
             return ResponseEntity.ok(
                     Arrays.stream(Objects.requireNonNull(destinationFolder.listFiles()))
                             .filter(f -> !f.getName().endsWith("tmp"))
+                            .filter(f -> f.getName().toLowerCase().endsWith(".jpg") || f.getName().toLowerCase().endsWith(".jpeg"))
                             .sorted(Comparator.comparing(File::getName))
                             .map(GalleryImage::fromFile)
                             .filter(Objects::nonNull)
@@ -97,6 +102,39 @@ public class GalleryEndpoint {
         }
         return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
     }
+
+    @GetMapping("/gallery/cache")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> getCacheSize() {
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    @GetMapping("/gallery/cache/clear")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> clearCache() {
+        thumbnailGenerator.clearCache();
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    private String humanReadableByteCount(long bytes, boolean si) {
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class CacheSize {
+
+    private String size;
+
+    public CacheSize(String size) {
+        this.size = size;
+    }
 }
 
 @Getter
@@ -104,13 +142,44 @@ public class GalleryEndpoint {
 @NoArgsConstructor
 class GalleryImage {
 
+    private static final CacheLoader<File, Dimension> loader = new CacheLoader<>() {
+
+        @Override
+        public Dimension load(File file) {
+            try (ImageInputStream in = ImageIO.createImageInputStream(file)) {
+                final Iterator<ImageReader> readers = ImageIO.getImageReaders(in);
+                if (readers.hasNext()) {
+                    ImageReader reader = readers.next();
+                    try {
+                        reader.setInput(in);
+                        return new Dimension(reader.getWidth(0), reader.getHeight(0));
+                    } finally {
+                        reader.dispose();
+                    }
+                } else {
+                    logger.error(String.format("No reader found for image %s", file.toString()));
+                    return null;
+                }
+            } catch (Exception e) {
+                logger.error(String.format("Failed to create image object for %s", file.toString()), e);
+                return null;
+            }
+        }
+    };
+
+    private static final LoadingCache<File, Dimension> cache = CacheBuilder.newBuilder()
+            .maximumSize(20000)
+            .build(loader);
+
     private static final Logger logger = LoggerFactory.getLogger(GalleryImage.class);
+    private String title;
     private String src;
     private String msrc;
     private double w;
     private double h;
 
-    public GalleryImage(String src, String msrc, double w, double h) {
+    public GalleryImage(String title, String src, String msrc, double w, double h) {
+        this.title = title;
         this.src = src;
         this.msrc = msrc;
         this.w = w;
@@ -118,26 +187,16 @@ class GalleryImage {
     }
 
     public static GalleryImage fromFile(File file) {
-        Dimension dimension;
-        try (ImageInputStream in = ImageIO.createImageInputStream(file)) {
-            final Iterator<ImageReader> readers = ImageIO.getImageReaders(in);
-            if (readers.hasNext()) {
-                ImageReader reader = readers.next();
-                try {
-                    reader.setInput(in);
-                    dimension = new Dimension(reader.getWidth(0), reader.getHeight(0));
-                } finally {
-                    reader.dispose();
-                }
-            } else {
-                logger.error(String.format("No reader found for image %s", file.toString()));
-                return null;
-            }
-        } catch (Exception e) {
-            logger.error(String.format("Failed to create image object for %s", file.toString()), e);
+
+        Dimension dimension = null;
+        try {
+            dimension = cache.get(file);
+        } catch (ExecutionException e) {
+            logger.error(String.format("Failed to get image dimensions for %s", file.toString()), e);
+        }
+        if (dimension == null) {
             return null;
         }
-
-        return new GalleryImage(file.getName(), file.getName(), dimension.getWidth(), dimension.getHeight());
+        return new GalleryImage(file.getName(), file.getName(), file.getName(), dimension.getWidth(), dimension.getHeight());
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java

@@ -9,11 +9,14 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.PostParseException;
-import tn.mnlr.vripper.q.DownloadQ;
+import tn.mnlr.vripper.q.ExecutionService;
 import tn.mnlr.vripper.services.*;
 
 import java.io.File;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -47,18 +50,20 @@ public class PostRestEndpoint {
     }
 
     @Autowired
-    private DownloadQ downloadQ;
+    private ExecutionService executionService;
+
 
     @Autowired
     private PostParser postParser;
 
     @PostMapping("/post")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity processPost(@RequestBody ThreadUrl _url) throws Exception {
+    public synchronized ResponseEntity processPost(@RequestBody ThreadUrl _url) throws Exception {
         if (_url.getUrl() == null || _url.getUrl().isEmpty()) {
             return new ResponseEntity("Failed to process empty request", HttpStatus.BAD_REQUEST);
         }
         List<String> urls = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
+        ArrayList<QueuedVGLink> queuedVGLinks = new ArrayList<>();
         for (String url : urls) {
             logger.debug(String.format("Starting to process thread: %s", url));
             if (!url.startsWith("https://vipergirls.to")) {
@@ -78,8 +83,9 @@ public class PostRestEndpoint {
             } catch (Exception e) {
                 throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
             }
-            vgHandler.handle(Collections.singletonList(new QueuedVGLink(url, threadId, postId)));
+            queuedVGLinks.add(new QueuedVGLink(url, threadId, postId));
         }
+        vgHandler.handle(queuedVGLinks);
         return ResponseEntity.ok().build();
     }
 
@@ -87,13 +93,13 @@ public class PostRestEndpoint {
     @ResponseStatus(value = HttpStatus.OK)
     public synchronized void restartPost(@RequestBody @NonNull List<PostId> postIds) throws Exception {
         for (PostId postId : postIds) {
-            downloadQ.restart(postId.getPostId());
+            executionService.restart(postId.getPostId());
         }
     }
 
     @PostMapping("/post/add")
     @ResponseStatus(value = HttpStatus.OK)
-    public void addPost(@RequestBody List<PostToAdd> posts) {
+    public synchronized void addPost(@RequestBody List<PostToAdd> posts) {
         for (PostToAdd post : posts) {
             VripperApplication.commonExecutor.submit(() -> {
                 try {
@@ -114,22 +120,22 @@ public class PostRestEndpoint {
 
     @PostMapping("/post/restart/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost() throws Exception {
-        downloadQ.restartAll();
+    public synchronized void restartPost() throws Exception {
+        executionService.restartAll();
     }
 
     @PostMapping("/post/stop")
     @ResponseStatus(value = HttpStatus.OK)
     public synchronized void stop(@RequestBody @NonNull List<PostId> postIds) {
         for (PostId postId : postIds) {
-            downloadQ.stop(postId.getPostId());
+            executionService.stop(postId.getPostId());
         }
     }
 
     @PostMapping("/post/stop/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stopAll() {
-        downloadQ.stopAll();
+    public synchronized void stopAll() {
+        executionService.stopAll();
     }
 
     @PostMapping("/post/remove")
@@ -137,7 +143,7 @@ public class PostRestEndpoint {
     public synchronized ResponseEntity<List<RemoveResult>> remove(@RequestBody @NonNull List<PostId> postIds) {
         List<RemoveResult> result = new ArrayList<>();
         for (PostId postId : postIds) {
-            downloadQ.stop(postId.getPostId());
+            executionService.stop(postId.getPostId());
             appStateService.remove(postId.getPostId());
             result.add(new RemoveResult(postId.getPostId()));
         }
@@ -146,13 +152,13 @@ public class PostRestEndpoint {
 
     @PostMapping("/post/clear/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity<RemoveAllResult> clearAll() {
+    public synchronized ResponseEntity<RemoveAllResult> clearAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.clearAll()));
     }
 
     @PostMapping("/post/remove/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity<RemoveAllResult> removeAll() {
+    public synchronized ResponseEntity<RemoveAllResult> removeAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.removeAll()));
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -60,6 +60,7 @@ public class SettingsRestEndpoint {
             }
             this.settings.setDownloadPath(settings.getDownloadPath());
             this.settings.setMaxThreads(settings.getMaxThreads());
+            this.settings.setMaxTotalThreads(settings.getMaxTotalThreads());
             this.settings.setAutoStart(settings.isAutoStart());
             this.settings.setVLogin(settings.isVLogin());
 
@@ -80,6 +81,7 @@ public class SettingsRestEndpoint {
             this.settings.setThreadSubLocation(settings.isThreadSubLocation());
             this.settings.setClearCompleted(settings.isClearCompleted());
             this.settings.setViewPhotos(settings.isViewPhotos());
+            this.settings.setNotificationEnabled(settings.isNotification());
 
             this.settings.save();
 
@@ -95,6 +97,7 @@ public class SettingsRestEndpoint {
         return new AppSettingsService.Settings(
                 settings.getDownloadPath(),
                 settings.getMaxThreads(),
+                settings.getMaxTotalThreads(),
                 settings.isAutoStart(),
                 settings.isVLogin(),
                 settings.getVUsername(),
@@ -105,7 +108,8 @@ public class SettingsRestEndpoint {
                 settings.isSubLocation(),
                 settings.isThreadSubLocation(),
                 settings.isClearCompleted(),
-                settings.isViewPhotos()
+                settings.isViewPhotos(),
+                settings.isNotificationEnabled()
         );
     }
 

vripper-server/src/main/resources/application.properties

@@ -6,5 +6,5 @@ server.port=${vripper.server.port:8080}
 management.endpoints.web.exposure.include=shutdown
 management.endpoint.shutdown.enabled=true
 endpoints.shutdown.enabled=true
-spring.profiles.active=portable
-#spring.profiles.active=installer
+#spring.profiles.active=portable
+spring.profiles.active=installer

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.8.0",
+  "version": "2.10.5",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -35,7 +35,7 @@
     "zone.js": "~0.8.26"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^0.803.21",
+    "@angular-devkit/build-angular": "^0.803.23",
     "@angular/cli": "~8.3.20",
     "@angular/compiler-cli": "^8.2.14",
     "@angular/language-service": "~8.2.14",
@@ -43,7 +43,7 @@
     "@types/jasminewd2": "~2.0.3",
     "@types/node": "~8.9.4",
     "codelyzer": "~4.5.0",
-    "electron": "^6.0.9",
+    "electron": "^7.1.7",
     "jasmine-core": "~2.99.1",
     "jasmine-spec-reporter": "~4.2.1",
     "karma": "^4.2.0",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.8.0</version>
+    <version>2.10.5</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.module.ts

@@ -1,3 +1,4 @@
+import { LinkCollectorService } from './link-collector.service';
 import { ContextMenuService } from './ctxt-menu.service';
 import { PostContextMenuComponent } from './posts/post-context-menu.component';
 import { PostsDataService } from './posts-data.service';
@@ -101,7 +102,8 @@ import { PhotoSwipeComponent } from './photo-swipe/photo-swipe.component';
     SharedService,
     SelectionService,
     PostsDataService,
-    ContextMenuService
+    ContextMenuService,
+    LinkCollectorService
   ],
   bootstrap: [AppComponent]
 })

vripper-ui/src/app/app.service.ts

@@ -26,8 +26,6 @@ export class AppService {
   private _settings: Settings;
   private _renderer: Renderer2;
 
-  // private settings$: Subject<Settings> = new Subject();
-
   get settings(): Settings {
     return { ...this._settings };
   }

vripper-ui/src/app/clipboard.service.ts

@@ -1,6 +1,6 @@
 import { Subject, Observable } from 'rxjs';
 import { ElectronService } from 'ngx-electron';
-import { Injectable, OnInit } from '@angular/core';
+import { Injectable } from '@angular/core';
 import { Clipboard } from 'electron';
 import { HttpClient } from '@angular/common/http';
 import { Settings } from './common/settings.model';

vripper-ui/src/app/common/settings.model.ts

@@ -1,6 +1,7 @@
 export interface Settings {
     downloadPath: string;
     maxThreads: number;
+    maxTotalThreads: number;
     autoStart: boolean;
     vLogin: boolean;
     vUsername: string;
@@ -8,4 +9,5 @@ export interface Settings {
     vThanks: boolean;
     desktopClipboard: boolean;
     viewPhotos: boolean;
-  }
+    notification: boolean;
+  }

vripper-ui/src/app/gallery/gallery.component.html

@@ -4,23 +4,37 @@
   </div>
   <mat-dialog-content fxFlex="grow">
     <div class="container">
-      <div class="masonry ">
-        <ng-container *ngFor="let img of images | async; let i=index">
-          <figure class="masonry-brick " fxLayout="row" fxLayoutAlign="center center"
-                  style="position: relative; cursor: pointer;">
-            <mat-progress-spinner *ngIf="!img._initialized" mode="indeterminate"
-                                  style="position: absolute;"></mat-progress-spinner>
-            <img (click)="openSlideshow(i)" (load)="loaded(img)" [src]="img.msrc"/>
+      <div class="masonry">
+        <ng-container *ngFor="let img of images | async; let i = index">
+          <figure class="masonry-brick">
+            <img
+              [style.opacity]="img._initialized ? 1 : 0"
+              [style.visibility]="img._initialized ? 'visible' : 'hidden'"
+              (click)="openSlideshow(i)"
+              (load)="loaded(img)"
+              [src]="img.msrc"
+            />
           </figure>
         </ng-container>
+
         <app-photo-swipe #photoSwipe></app-photo-swipe>
       </div>
     </div>
   </mat-dialog-content>
+  <div style="height: 5px; width: 100%; margin-top: 8px">
+    <div *ngIf="loadProgress | async as progress">
+      <mat-progress-bar
+        *ngIf="!progress.done"
+        [mode]="progress.loading ? 'determinate' : 'indeterminate'"
+        [value]="progress.progress"
+      ></mat-progress-bar>
+    </div>
+  </div>
   <mat-dialog-actions align="end" fxFlex="nogrow">
-    <button (click)="refresh()" color="primary" mat-raised-button>
+    <button (click)="refresh()" [disabled]="disableRefresh | async" color="primary" mat-raised-button>
       <mat-icon>refresh</mat-icon>
-      <span>Refresh</span></button>
+      <span>Refresh</span>
+    </button>
     <button mat-dialog-close mat-raised-button>Close</button>
   </mat-dialog-actions>
 </div>

vripper-ui/src/app/gallery/gallery.component.scss

@@ -3,23 +3,22 @@
 }
 
 .masonry {
-  display: flex;
-  flex-flow: row wrap;
-  margin-left: -8px;
   width: 100%;
 }
 
 .masonry-brick {
-  flex: auto;
   height: 250px;
-  min-width: 150px;
-  margin: 0 8px 8px 0;
+  width: 250px;
+  display: inline-block;
+  cursor: pointer;
+  margin: 0 3px;
 }
 
 img {
+  opacity: 0;
+  visibility: hidden;
+  transition: opacity 0.5s ease-in, visibility 0.5s;
+  object-fit: cover;
   height: 250px;
-  width: auto;
-}
-
-figure {
+  width: 250px;
 }

vripper-ui/src/app/gallery/gallery.component.ts

@@ -7,11 +7,26 @@ import { PostState } from '../posts/post-state.model';
 import { BehaviorSubject, Subject } from 'rxjs';
 
 class Image extends IImage {
-  constructor(public src: string, public msrc: string, public w: number, public h: number, public _initialized: boolean) {
+  constructor(
+    public title: string,
+    public src: string,
+    public msrc: string,
+    public w: number,
+    public h: number,
+    public _initialized: boolean
+  ) {
     super(src, msrc, w, h);
   }
 }
 
+interface Progress {
+  loading: boolean;
+  progress: number;
+  done: boolean;
+}
+
+const initialProgressState: Progress = { loading: false, progress: 0, done: false };
+
 @Component({
   selector: 'app-gallery',
   templateUrl: './gallery.component.html',
@@ -30,11 +45,23 @@ export class GalleryComponent implements OnInit, OnDestroy {
 
   images: Subject<Image[]> = new BehaviorSubject([]);
   _images: Image[] = [];
+  disableRefresh: Subject<boolean> = new BehaviorSubject(true);
+  loadProgress: Subject<Progress> = new BehaviorSubject(initialProgressState);
+  loadedImages = 0;
 
   @ViewChild('photoSwipe', { static: true }) photoSwipe: PhotoSwipeComponent;
 
   loaded(img: Image) {
     img._initialized = true;
+    this.loadedImages++;
+    this.loadProgress.next({
+      loading: true,
+      progress: Math.floor((this.loadedImages / this._images.length) * 100),
+      done: this.loadedImages === this._images.length
+    });
+    if (this.loadedImages === this._images.length) {
+      this.disableRefresh.next(false);
+    }
   }
 
   ngOnInit() {
@@ -62,6 +89,9 @@ export class GalleryComponent implements OnInit, OnDestroy {
   }
 
   refresh() {
+    this.loadProgress.next(initialProgressState);
+    this.loadedImages = 0;
+    this.disableRefresh.next(true);
     this.httpClient.get<Image[]>(this.serverService.baseUrl + '/gallery/' + this.dialogData.postId).subscribe(
       response => {
         this.ngZone.run(() => {

vripper-ui/src/app/grab-queue/grab-queue.component.ts

@@ -1,16 +1,23 @@
+import { LinkCollectorService } from './../link-collector.service';
 import { UrlGrabRendererComponent } from './url-renderer.component';
 import { GrabQueueDataSource } from './grab-queue.datasource';
-import { Component, OnInit, NgZone } from '@angular/core';
+import { Component, OnInit, NgZone, ChangeDetectionStrategy } from '@angular/core';
 import { GridOptions } from 'ag-grid-community';
 import { WsConnectionService } from '../ws-connection.service';
+import { NotificationService } from '../notification.service';
 
 @Component({
   selector: 'app-grab-queue',
   templateUrl: './grab-queue.component.html',
-  styleUrls: ['./grab-queue.component.scss']
+  styleUrls: ['./grab-queue.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
 export class GrabQueueComponent implements OnInit {
-  constructor(private wsConnection: WsConnectionService, private zone: NgZone) {
+  constructor(
+    private wsConnection: WsConnectionService,
+    private zone: NgZone,
+    private linkCollectorService: LinkCollectorService,
+    private notificationService: NotificationService) {
     this.gridOptions = <GridOptions>{
       columnDefs: [
         {
@@ -33,11 +40,14 @@ export class GrabQueueComponent implements OnInit {
       getRowNodeId: data => data['link'],
       onGridReady: () => {
         this.gridOptions.api.sizeColumnsToFit();
-        this.dataSource = new GrabQueueDataSource(this.wsConnection, this.gridOptions, this.zone);
+        this.dataSource = new GrabQueueDataSource(this.wsConnection, this.gridOptions, this.zone, this.notificationService);
         this.dataSource.connect();
       },
       onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
-      onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit()
+      onRowDataUpdated: () => {
+        this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount());
+        this.gridOptions.api.sizeColumnsToFit();
+      }
     };
   }
 

vripper-ui/src/app/grab-queue/grab-queue.datasource.ts

@@ -3,6 +3,7 @@ import { CMD } from './../common/cmd.enum';
 import { WSMessage } from './../common/ws-message.model';
 import { Subscription } from 'rxjs';
 import { WsConnectionService } from '../ws-connection.service';
+import { NotificationService } from '../notification.service';
 import { GridOptions } from 'ag-grid-community';
 import { WsHandler } from '../ws-handler';
 import { NgZone } from '@angular/core';
@@ -11,7 +12,8 @@ export class GrabQueueDataSource {
   constructor(
     private wsConnectionService: WsConnectionService,
     private gridOptions: GridOptions,
-    private zone: NgZone
+    private zone: NgZone,
+    private notificationService: NotificationService
   ) {
     this.websocketHandlerPromise = this.wsConnectionService.getConnection();
   }
@@ -43,6 +45,13 @@ export class GrabQueueDataSource {
               }
             });
             this.gridOptions.api.updateRowData({ update: toUpdate, add: toAdd, remove: toRemove });
+            const count = this.gridOptions.api.getDisplayedRowCount();
+            if (count > 0 && toAdd.length > 0) {
+              this.notificationService.notifyFromGrabQueue(
+                'Link Collector',
+                `You have ${count} ${count > 1 ? 'threads' : 'thread'} waiting in the link collector`
+              );
+            }
           });
         })
       );

vripper-ui/src/app/home/home.component.html

@@ -2,7 +2,13 @@
   <mat-tab label="Downloads">
     <app-posts style="width: 100%; height: 100%;"></app-posts>
   </mat-tab>
-  <mat-tab label="Link Collector">
+  <mat-tab>
+    <ng-template mat-tab-label>
+      <span style="margin-right: 5px;">Link Collector</span>
+      <ng-container *ngIf="linkCollectorService.count | async as count">
+        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count">link</mat-icon>
+      </ng-container>
+    </ng-template>
     <app-grab-queue style="width: 100%; height: 100%;"></app-grab-queue>
   </mat-tab>
 </mat-tab-group>

vripper-ui/src/app/home/home.component.ts

@@ -1,3 +1,4 @@
+import { LinkCollectorService } from './../link-collector.service';
 import { ServerService } from './../server-service';
 import { ElectronService } from 'ngx-electron';
 import { ClipboardService } from './../clipboard.service';
@@ -19,7 +20,8 @@ export class HomeComponent implements OnInit, OnDestroy {
     private httpClient: HttpClient,
     private serverService: ServerService,
     private _snackBar: MatSnackBar,
-    private ngZone: NgZone
+    private ngZone: NgZone,
+    public linkCollectorService: LinkCollectorService
   ) {}
 
   ngOnInit() {

vripper-ui/src/app/link-collector.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@angular/core';
+import { Subject, Observable, BehaviorSubject } from 'rxjs';
+
+@Injectable()
+export class LinkCollectorService {
+
+    private _count$: Subject<number> = new BehaviorSubject(0);
+
+    get count(): Observable<number> {
+        return this._count$.asObservable();
+    }
+
+    setCount(count: number) {
+        this._count$.next(count);
+    }
+}

vripper-ui/src/app/material.module.ts

@@ -17,7 +17,8 @@ import {
     MatDividerModule,
     MatChipsModule,
     MatListModule,
-    MatProgressSpinnerModule
+    MatProgressSpinnerModule,
+    MatBadgeModule
 } from '@angular/material';
 
 @NgModule({
@@ -38,7 +39,8 @@ import {
         MatDividerModule,
         MatChipsModule,
         MatListModule,
-        MatProgressSpinnerModule
+        MatProgressSpinnerModule,
+        MatBadgeModule
     ],
     providers: [MatSnackBar]
 })

vripper-ui/src/app/notification.service.ts

@@ -0,0 +1,26 @@
+import { Injectable } from '@angular/core';
+import { AppService } from './app.service';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class NotificationService {
+  constructor(private appService: AppService) {}
+
+  notifyFromGrabQueue(title: string, body: string) {
+    if (!this.appService.settings.notification) {
+      return;
+    }
+    if (!('Notification' in window)) {
+      return;
+    } else if (Notification.permission === 'granted') {
+      const notification = new Notification(title, { body: body, icon: 'assets/icon.png' });
+    } else if (Notification.permission !== 'denied') {
+      Notification.requestPermission().then(function(permission) {
+        if (permission === 'granted') {
+          const notification = new Notification(title, { body: body, icon: 'assets/icon.png' });
+        }
+      });
+    }
+  }
+}

vripper-ui/src/app/post-detail/post-details-progress.component.scss-theme.scss

@@ -13,7 +13,7 @@
   }
 
   app-details-cell .error {
-    background-color: mat-color($red, 50);
+    background-color: mat-color($red, 100);
     & .progress-bar .mat-progress-bar-fill::after {
       background-color: mat-color($red, 300);
     }

vripper-ui/src/app/posts/post-progress.renderer.component.html

@@ -3,7 +3,8 @@
     <div
       [ngClass]="{
         error: postState.status === 'ERROR',
-        downloading: postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL',
+        downloading: postState.status === 'DOWNLOADING',
+        partial: postState.status === 'PARTIAL',
         complete: postState.status === 'COMPLETE',
         stopped: postState.status === 'STOPPED'
       }"

vripper-ui/src/app/posts/post-progress.renderer.component.scss-theme.scss

@@ -4,6 +4,7 @@
   $red: mat-palette($mat-red);
   $grey: mat-palette($mat-grey);
   $green: mat-palette($mat-green);
+  $orange: mat-palette($mat-orange);
 
   app-progress-cell .progress-bar .mat-progress-bar-fill::after {
     background-color: mat-color($grey, 400);
@@ -14,7 +15,7 @@
   }
 
   app-progress-cell .error {
-    background-color: mat-color($red, 50);
+    background-color: mat-color($red, 100);
     & .progress-bar .mat-progress-bar-fill::after {
       background-color: mat-color($red, 300);
     }
@@ -23,6 +24,16 @@
     }
   }
 
+  app-progress-cell .partial {
+    background-color: mat-color($orange, 100);
+    & .progress-bar .mat-progress-bar-fill::after {
+      background-color: mat-color($orange, 300);
+    }
+    & .progress-bar .mat-progress-bar-buffer {
+      background-color: mat-color($orange, 100);
+    }
+  }
+
   app-progress-cell .downloading {
     background-color: mat-color($green, 50);
     & .progress-bar .mat-progress-bar-fill::after {
@@ -44,6 +55,6 @@
   }
 
   app-progress-cell .stopped {
-    background-color: mat-color($grey, 100);
+    background-color: mat-color($grey, 300);
   }
 }

vripper-ui/src/app/settings/settings.component.html

@@ -25,18 +25,32 @@
               </button>
             </div>
 
-            <mat-form-field>
-              <input
-                formControlName="maxThreads"
-                type="number"
-                matInput
-                placeholder="Max concurrent downloads"
-                name="maxThreads"
-                min="1"
-                max="4"
-                required
-              />
-            </mat-form-field>
+            <div fxLayout="row" fxLayoutAlign="center center" fxLayoutGap="10px">
+              <mat-form-field fxFlex="grow">
+                <input
+                  formControlName="maxThreads"
+                  matInput
+                  max="4"
+                  min="1"
+                  name="maxThreads"
+                  placeholder="Max concurrent downloads per host"
+                  required
+                  type="number"
+                />
+              </mat-form-field>
+
+              <mat-form-field fxFlex="grow">
+                <input
+                  formControlName="maxTotalThreads"
+                  matInput
+                  min="0"
+                  name="maxTotalThreads"
+                  placeholder="Max global concurrent downloads"
+                  required
+                  type="number"
+                />
+              </mat-form-field>
+            </div>
 
             <mat-checkbox color="primary" formControlName="subLocation" name="subLocation"
             >Save posts in sub folders
@@ -64,8 +78,15 @@
             </mat-checkbox>
 
             <mat-checkbox color="primary" formControlName="viewPhotos" name="viewPhotos">
-              Enable View photo in gallery option
+              Enable photo gallery
             </mat-checkbox>
+            <div style="margin-left: 15px">
+              Cache size: {{(cacheSize | async).size}}
+              <button (click)="clearCache()" [disabled]="(cacheClearLoading | async)" color="primary"
+                      mat-stroked-button>
+                <mat-icon>clear</mat-icon>
+                <span>Clear</span></button>
+            </div>
 
             <mat-slide-toggle color="primary" formControlName="vLogin" name="vLogin">
               ViperGirls Authentication
@@ -92,10 +113,17 @@
             </section>
           </form>
         </mat-tab>
-        <mat-tab label="Desktop Integration" *ngIf="electronService.isElectronApp">
+        <mat-tab label="Desktop Integration">
           <form [formGroup]="desktopSettingsForm" autocomplete="off">
-            <mat-checkbox color="primary" formControlName="desktopClipboard" name="desktopClipboard"
-              >Monitor Clipboard</mat-checkbox
+            <mat-checkbox *ngIf="electronService.isElectronApp" color="primary" formControlName="desktopClipboard"
+                          name="desktopClipboard"
+            >Monitor Clipboard
+            </mat-checkbox
+            >
+
+            <mat-checkbox color="primary" formControlName="notification" name="notification"
+            >Enable system notifications
+            </mat-checkbox
             >
           </form>
         </mat-tab>

vripper-ui/src/app/settings/settings.component.ts

@@ -9,6 +9,11 @@ import { ServerService } from '../server-service';
 import { ElectronService } from 'ngx-electron';
 import { Settings } from '../common/settings.model';
 import { OpenDialogReturnValue } from 'electron';
+import { Subject, BehaviorSubject } from 'rxjs';
+
+interface CacheSize {
+  size: string;
+}
 
 @Component({
   selector: 'app-settings',
@@ -31,6 +36,7 @@ export class SettingsComponent implements OnInit {
   generalSettingsForm = new FormGroup({
     downloadPath: new FormControl(''),
     maxThreads: new FormControl(''),
+    maxTotalThreads: new FormControl(''),
     autoStart: new FormControl(false),
     forceOrder: new FormControl(false),
     subLocation: new FormControl(false),
@@ -44,10 +50,13 @@ export class SettingsComponent implements OnInit {
   });
 
   desktopSettingsForm = new FormGroup({
-    desktopClipboard: new FormControl(false)
+    desktopClipboard: new FormControl(false),
+    notification: new FormControl(false)
   });
 
   darkTheme = false;
+  cacheSize: Subject<CacheSize> = new BehaviorSubject({size: '0'});
+  cacheClearLoading: Subject<boolean> = new BehaviorSubject(false);
 
   updateTheme() {
     this.appService.updateTheme(this.darkTheme);
@@ -59,17 +68,36 @@ export class SettingsComponent implements OnInit {
 
   ngOnInit() {
     this.darkTheme = this.appService.darkTheme;
-    this.httpClient.get<Settings>(this.serverService.baseUrl + '/settings').subscribe(
-      data => {
-        this.generalSettingsForm.reset(data);
-        this.desktopSettingsForm.reset(data);
-      },
-      error => {
-        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
-          duration: 5000
-        });
-      }
-    );
+    this.httpClient.get<Settings>(this.serverService.baseUrl + '/settings')
+    .subscribe(data => {
+      this.generalSettingsForm.reset(data);
+      this.desktopSettingsForm.reset(data);
+    }, error => {
+      this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+        duration: 5000
+      });
+    });
+    this.httpClient.get<CacheSize>(this.serverService.baseUrl + '/gallery/cache')
+    .subscribe(data => {
+      this.cacheSize.next(data);
+    }, error => {
+      this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+        duration: 5000
+      });
+    });
+  }
+
+  clearCache() {
+    this.cacheClearLoading.next(true);
+    this.httpClient.get<CacheSize>(this.serverService.baseUrl + '/gallery/cache/clear')
+    .pipe(finalize(() => this.cacheClearLoading.next(false)))
+    .subscribe(data => {
+      this.cacheSize.next(data);
+    }, error => {
+      this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+        duration: 5000
+      });
+    })
   }
 
   browse() {

vripper-ui/src/app/ws-connection.service.ts

@@ -27,6 +27,10 @@ export class WsConnectionService {
   private wsHandler: WsHandler;
 
   constructor(private electronService: ElectronService, private serverService: ServerService) {
+    this.init();
+  }
+
+  init() {
     this.wsHandlerPromise = new Promise((resolve, reject) => {
       if (this.wsHandler != null) {
         return this.wsHandler;

vripper-ui/src/environments/environment.prod.ts

@@ -1,5 +1,5 @@
 export const environment = {
   production: true,
   localhost: '',
-  version: '2.8.0'
+  version: '2.10.5'
 };

vripper-ui/src/environments/environment.ts

@@ -5,7 +5,7 @@
 export const environment = {
   production: false,
   localhost: 'http://localhost:8080',
-  version: '2.8.0'
+  version: '2.10.5'
 };
 
 /*