v2.10.5

Add system notification
Bug fixes

CHANGELOG.md

@@ -1,5 +1,109 @@
 # Changelog
 
+## [2.10.5] - 2020-01-18
+### Changed
+- Bug fixes
+
+## [2.10.4] - 2020-01-18
+### Changed
+- Zero value for total max download will disable the limit
+- Add system notification
+- Bug fixes
+
+## [2.10.3] - 2020-01-16
+### Changed
+- Some bug fixes
+- Enhance download queue logic
+- Clear cache button
+- Fix the partial status
+- Sanitize URLs with spaces
+### Added
+- Support for Postimg host
+- Support for Imagevenue host
+
+## [2.10.2] - 2019-12-29
+### Changed
+- Upgrade electron to v7
+- Bug fix
+
+## [2.10.1] - 2019-12-27
+### Changed
+- Bug fix
+
+## [2.10.0] - 2019-12-27
+### Changed
+- Fix spring dependency bug
+### Added
+- Add support for imgspice
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Fix build issue
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Decrease timeout for fast fail
+### Added
+- Allow many hosts to run at the same time
+- Add title to gallery
+- Add global concurrent downloads
+- Add counter badge in link collector
+
+## [2.8.1] - 2019-12-24
+### Changed
+- Fix issue with path creation
+
+## [2.8.0] - 2019-12-24
+### Added
+- Photo gallery feature
+
+## [2.7.1] - 2019-12-21
+### Added
+- Add version to title bar
+
+## [2.7.0] - 2019-12-21
+### Changed
+- Add missing icon
+### Added
+- Add support for pixxxels
+
+## [2.6.0] - 2019-12-21
+### Changed
+- Disable download button when nothing selected in Link collector grab menu
+- Better quality app icon
+- Implement context menu on download tab
+- Add stop remove and start button to context menu
+- Link will not be removed automatically from link collector tab
+- Some UI fixes
+
+## [2.5.1] - 2019-12-09
+### Changed
+- Disable download button when nothing selected in Link collector grab menu
+### Added
+- Add hosts column in Link collector grab menu
+
+## [2.5.0] - 2019-12-08
+### Changed
+- Change loading screen
+- UI Fixes
+- Fix bug with previews
+### Added
+- Add link collector feature
+- Add support for multiple links at a time
+
+## [2.4.1] - 2019-11-26
+### Changed
+- Increase attempts for establishing ws connection between front and back end
+- Change default download folder to Home folder (installer), App folder (portable)
+
+## [2.4.0] - 2019-11-23
+### Changed
+- Pimpandhost Host support
+
+## [2.3.2] - 2019-11-23
+### Changed
+- Fix a bug with the scan component
+
 ## [2.3.1] - 2019-11-22
 ### Changed
 - Fix connexion reset issue

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.3.1</version>
+    <version>2.10.5</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/main.js

@@ -23,6 +23,11 @@ process.on("uncaughtException", err => {
 });
 
 function createWindow() {
+
+  if (process.platform === 'win32') {
+    app.setAppUserModelId("tn.mnlr.vripper");
+  }
+
   let icon;
   if(process.platform === "win32") {
     icon = __dirname + '/icon.ico';
@@ -59,52 +64,51 @@ function createWindow() {
   });
 }
 
-getPort().then(port => {
-  serverPort = port;
-  ipcMain.on("get-port", event => {
-    event.reply("port", port);
-  });
-  let javaBinPath;
-  if(appDir !== undefined) {
-    javaBinPath = path.join(appDir, "java-runtime/bin/java");
-  } else {
-    if(process.platform === 'darwin') {
-      javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
-    } else {
-      javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
-    }
-  }
-  let jarPath;
-  if(appDir !== undefined) {
-    jarPath = path.join(appDir, "bin/vripper-server.jar");
-  } else {
-    if(process.platform === 'darwin') {
-      jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
-    } else {
-      jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
-    }
-  }
-  vripperServer = spawn(javaBinPath, [
-    "-Xms256m",
-    "-Xmx1024m",
-    "-Dvripper.server.port=" + port,
-    "-jar",
-    jarPath
-  ], {
-    stdio: 'ignore'
-  });
-  vripperServer.on('exit', (code, signal) => {
-    console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
-    terminated = true;
-    app.quit();
-  });
-});
-
 const gotTheLock = app.requestSingleInstanceLock();
 
 if (!gotTheLock) {
   app.quit();
 } else {
+  getPort().then(port => {
+    serverPort = port;
+    ipcMain.on("get-port", event => {
+      event.reply("port", port);
+    });
+    let javaBinPath;
+    if(appDir !== undefined) {
+      javaBinPath = path.join(appDir, "java-runtime/bin/java");
+    } else {
+      if(process.platform === 'darwin') {
+        javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+      } else {
+        javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+      }
+    }
+    let jarPath;
+    if(appDir !== undefined) {
+      jarPath = path.join(appDir, "bin/vripper-server.jar");
+    } else {
+      if(process.platform === 'darwin') {
+        jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+      } else {
+        jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+      }
+    }
+    vripperServer = spawn(javaBinPath, [
+      "-Xms256m",
+      "-Dvripper.server.port=" + port,
+      "-jar",
+      jarPath
+    ], {
+      stdio: 'ignore'
+    });
+    vripperServer.on('exit', (code, signal) => {
+      console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
+      terminated = true;
+      app.quit();
+    });
+  });
+  
   app.on("second-instance", (event, commandLine, workingDirectory) => {
     if (win) {
       if (win.isMinimized()) win.restore();
@@ -136,6 +140,7 @@ if (!gotTheLock) {
       })
       .catch((error) => {
         // Terminate immediately
+        console.log(error);
         vripperServer.kill('SIGKILL');
         terminated = true;
         app.quit();

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.3.1",
+  "version": "2.10.5",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -8,7 +8,7 @@
   "license": "ISC",
   "build": {
     "appId": "tn.mnlr.vripper",
-    "productName": "vripper",
+    "productName": "VRipper",
     "files": [
       "**/*",
       "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -38,7 +38,7 @@
       }
     ],
     "win": {
-      "icon": "icon.png",
+      "icon": "icon.ico",
       "target": [
         "nsis"
       ]
@@ -62,7 +62,7 @@
       "target": [
         "dmg"
       ],
-      "icon": "icon.png"
+      "icon": "icon.icns"
     }
   },
   "scripts": {
@@ -70,14 +70,13 @@
     "dist": "node pre-build.js && electron-builder"
   },
   "devDependencies": {
-    "electron": "^6.0.9",
-    "electron-builder": "^21.2.0"
+    "electron": "^7.1.9",
+    "electron-builder": "^22.2.0"
   },
   "dependencies": {
     "axios": "^0.19.0",
     "cheerio": "^1.0.0-rc.3",
     "copy-dir": "^1.1.0",
-    "custom-electron-titlebar": "^3.0.10",
     "electron-context-menu": "^0.13.0",
     "get-port": "^5.0.0",
     "rimraf": "^2.6.3"

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.3.1</version>
+    <version>2.10.5</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>2.3.1</version>
+        <version>2.10.5</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
@@ -67,6 +67,16 @@
             <artifactId>failsafe</artifactId>
             <version>2.0.1</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>28.1-jre</version>
+        </dependency>
+        <dependency>
+            <groupId>org.imgscalr</groupId>
+            <artifactId>imgscalr-lib</artifactId>
+            <version>4.2</version>
+        </dependency>
         <dependency>
             <groupId>tn.mnlr</groupId>
             <artifactId>vripper-ui</artifactId>

vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java

@@ -0,0 +1,24 @@
+package tn.mnlr.vripper;
+
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.event.ContextRefreshedEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+import tn.mnlr.vripper.services.PersistenceService;
+
+@Component
+public class EventListenerBean {
+
+    @Autowired
+    private PersistenceService persistenceService;
+
+    @Getter
+    private static boolean init = false;
+
+    @EventListener
+    public void onApplicationEvent(ContextRefreshedEvent event) {
+        init = true;
+        persistenceService.restore();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/SpringContext.java

@@ -27,7 +27,9 @@ public class SpringContext implements ApplicationContextAware {
 
     public static void close() {
         logger.info("Application terminating...");
-        context.close();
+        if (context != null) {
+            context.close();
+        }
 
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java

@@ -1,21 +1,10 @@
 package tn.mnlr.vripper;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.stereotype.Component;
-import tn.mnlr.vripper.exception.VripperException;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.PersistenceService;
-import tn.mnlr.vripper.services.VipergirlsAuthService;
 
-import javax.annotation.PostConstruct;
-import java.io.File;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -27,51 +16,11 @@ public class VripperApplication {
 
     public static void main(String[] args) {
         try {
+            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
             SpringApplication.run(VripperApplication.class, args);
         } catch (Exception e) {
             logger.error("Failed to run the application", e);
         }
     }
-
-    @Component
-    public class AppCommandRunner implements CommandLineRunner {
-
-        @Autowired
-        private VipergirlsAuthService authService;
-
-        @Autowired
-        private PersistenceService persistenceService;
-
-        @Autowired
-        private AppSettingsService appSettingsService;
-
-        @Value("${base.dir}")
-        private String baseDir;
-
-        @Getter
-        private String dataPath;
-
-        @PostConstruct
-        public void init() {
-            dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
-        }
-
-        @Override
-        public void run(String... args) {
-
-            persistenceService.restore();
-            appSettingsService.restore();
-            registerShutdownHook();
-
-            try {
-                authService.authenticate();
-            } catch (VripperException e) {
-                logger.error("Cannot authenticate user with ViperGirls", e);
-            }
-        }
-
-        private void registerShutdownHook() {
-            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
-        }
-    }
 }
+

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -27,8 +27,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.Iterator;
+import java.util.Objects;
 
 @Service
 abstract public class Host {
@@ -62,6 +66,11 @@ abstract public class Host {
 
     abstract public String getLookup();
 
+    @Override
+    public int hashCode() {
+        return Objects.hash(pathService);
+    }
+
     public boolean isSupported(String url) {
         return url.contains(getLookup());
     }
@@ -95,11 +104,10 @@ abstract public class Host {
             imageFileData.setImageName(formatImageFileName);
             File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
             logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
+            logger.debug(String.format("Creating %s", destinationFolder.getPath()));
+            destinationFolder.mkdirs();
             if (!destinationFolder.exists()) {
-                logger.debug(String.format("Creating %s", destinationFolder.getPath()));
-                if (destinationFolder.mkdirs()) {
-                    logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
-                }
+                logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
             }
 
             HttpClient client = cm.getClient().build();
@@ -112,7 +120,7 @@ abstract public class Host {
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
-                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
+                File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
                 try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
                     image.setTotal(response.getEntity().getContentLength());
                     logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
@@ -125,9 +133,11 @@ abstract public class Host {
                         image.increase(read);
                         downloadSpeedService.increase(read);
                     }
+                    fos.flush();
                     EntityUtils.consumeQuietly(response.getEntity());
                 }
-                checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
+                File finalName = checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
+                imageFileData.setFileName(finalName.getName());
             }
         } catch (Exception e) {
             if (Thread.interrupted()) {
@@ -137,27 +147,38 @@ abstract public class Host {
         }
     }
 
-    private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
-        String formatName;
+    private File checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
         try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
             Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
             if (!it.hasNext()) {
                 throw new HostException("Image file is not recognized!");
             }
             ImageReader reader = it.next();
-            formatName = reader.getFormatName();
-            if (formatName.toUpperCase().equals("JPEG")) {
-                formatName = "jpg";
+            if (reader.getFormatName().toUpperCase().equals("JPEG")) {
+                String imageNameLC = imageName.toLowerCase();
+                if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
+                    imageName += ".jpg";
+                } else {
+                    String toReplace = null;
+                    if (imageNameLC.endsWith("_jpg")) {
+                        toReplace = "_jpg";
+                    } else if (imageNameLC.endsWith("_jpeg")) {
+                        toReplace = "_jpeg";
+                    }
+                    if (toReplace != null) {
+                        imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
+                    }
+                }
             }
         } catch (Exception e) {
             throw new HostException("Failed to guess image format", e);
         }
         try {
-            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
+            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName);
             if (outImage.exists() && outImage.delete()) {
                 logger.debug(String.format("%s is deleted", outImage.toString()));
             }
-            Files.move(outputFile.toPath(), outImage.toPath());
+            return Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
         } catch (Exception e) {
             throw new HostException("Failed to rename the image", e);
         }
@@ -196,6 +217,20 @@ abstract public class Host {
         }
     }
 
+    protected String appendUri(String uri, String appendQuery) throws URISyntaxException {
+        URI oldUri = new URI(uri);
+
+        String newQuery = oldUri.getQuery();
+        if (newQuery == null) {
+            newQuery = appendQuery;
+        } else {
+            newQuery += "&" + appendQuery;
+        }
+
+        return new URI(oldUri.getScheme(), oldUri.getAuthority(),
+                oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
+    }
+
     private void setImageRequest(final ImageFileData imageFileData) {
         HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
         httpGet.addHeader("Referer", imageFileData.getPageUrl());
@@ -204,8 +239,17 @@ abstract public class Host {
 
     protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Host host = (Host) o;
+        return Objects.equals(getHost(), host.getHost());
+    }
+
     @Getter
     public static class Response {
+
         protected Response(Document document, Header[] headers) {
             this.document = document;
             this.headers = headers;

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageVenueHost.java

@@ -0,0 +1,79 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.net.URI;
+
+@Service
+public class ImageVenueHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImageVenueHost.class);
+
+    private static final String host = "imagevenue.com";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@id='thepic']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        //Sadly, they do not support https.
+        //If they add such support in the future,
+        //then we should automatically adapt the URL here as done elsewhere.
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            if(xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
+                //Button detected. No need to actually click it, just make the call again.
+                resp = getResponse(url, context);
+                doc = resp.getDocument();
+            }
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        if (imgNode == null) {
+            throw new HostException("Failed to locate image");
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            URI baseUri = new URI(url);
+            imageFileData.setImageUrl(new URI(baseUri.getScheme(), baseUri.getHost(), '/'+imgUrl, null).toString());
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch(Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgSpiceHost.java

@@ -0,0 +1,62 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class ImgSpiceHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImgSpiceHost.class);
+
+    private static final String host = "imgspice.com";
+    private static final String IMG_XPATH = "//img[@id='imgpreview']";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PimpandhostHost.java

@@ -0,0 +1,68 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class PimpandhostHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PimpandhostHost.class);
+
+    private static final String host = "pimpandhost.com";
+    private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url;
+        try {
+            url = appendUri(_url.replace("http://", "https://"), "size=original");
+        } catch (Exception e) {
+            throw new HostException(e);
+        }
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PixxxelsHost.java

@@ -0,0 +1,67 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class PixxxelsHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PixxxelsHost.class);
+
+    private static final String host = "pixxxels.cc";
+    private static final String IMG_XPATH = "//*[@id='download']";
+    private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = titleNode.getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PostImgHost.java

@@ -0,0 +1,61 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.util.Optional;
+
+@Service
+public class PostImgHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PostImgHost.class);
+
+    private static final String host = "postimg.cc";
+    private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
+    private static final String IMG_XPATH = "//a[@id='download']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Document doc = getResponse(url, context).getDocument();
+
+        Node urlNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            urlNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> getDefaultImageName(url));
+
+            imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
+            imageFileData.setImageName(imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java

@@ -1,156 +1,83 @@
 package tn.mnlr.vripper.q;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
 
-import java.util.Arrays;
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
+import java.util.Map;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingDeque;
 
 @Service
 public class DownloadQ {
 
-    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
-
     private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
 
     @Autowired
     private AppStateService appStateService;
 
+    private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> downloadQ = new ConcurrentHashMap<>();
     @Autowired
-    private ExecutionService executionService;
+    private AppSettingsService appSettingsService;
+    @Autowired
+    private List<Host> hosts;
 
-    private BlockingQueue<DownloadJob> downloadQ = new LinkedBlockingQueue<>();
-
-    @Getter
-    private boolean notPauseQ = true;
+    @PostConstruct
+    private void init() {
+        hosts.forEach(host -> downloadQ.put(host, new LinkedBlockingDeque<>()));
+    }
 
     public void put(Image image) throws InterruptedException {
-        synchronized (appStateService) {
-            logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
-            image.init();
-            DownloadJob downloadJob = new DownloadJob(image);
-            downloadQ.put(downloadJob);
-            appStateService.newDownloadJob(downloadJob);
+        logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
+        image.init();
+        DownloadJob downloadJob = new DownloadJob(image);
+        downloadQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
+        appStateService.newDownloadJob(downloadJob);
+    }
+
+    public void remove(final DownloadJob downloadJob) {
+        downloadQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
+    }
+
+    public List<DownloadJob> peek() {
+        List<DownloadJob> downloadJobs = new ArrayList<>();
+        if (hosts.size() == 0) {
+            return downloadJobs;
         }
+        for (Host host : hosts) {
+            Iterator<DownloadJob> it = downloadQ.get(host).iterator();
+            for (int i = 0; i < appSettingsService.getMaxThreads(); i++) {
+                DownloadJob downloadJob = it.hasNext() ? it.next() : null;
+                if (downloadJob != null) {
+                    downloadJobs.add(downloadJob);
+                }
+            }
+        }
+        return downloadJobs;
     }
 
-    DownloadJob take() throws InterruptedException {
-        DownloadJob downloadJob = downloadQ.take();
-        logger.debug(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
-        return downloadJob;
-    }
-
-    public synchronized void enqueue(Post post) throws InterruptedException {
+    public void enqueue(Post post) throws InterruptedException {
         for (Image image : post.getImages()) {
             put(image);
         }
     }
 
-    public void restart(String postId) throws InterruptedException {
-        synchronized (appStateService) {
-            if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
-                logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
-                return;
-            }
-            List<Image> images = appStateService.getPost(postId)
-                    .getImages()
-                    .stream()
-                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                    .collect(Collectors.toList());
-            if (images.isEmpty()) {
-                return;
-            }
-            appStateService.getPost(postId).setStatus(Post.Status.PENDING);
-            logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
-            for (Image image : images) {
-                put(image);
-            }
-        }
-    }
-
-    private void removeScheduled(Image image) {
-        synchronized (appStateService) {
-            image.setStatus(Image.Status.STOPPED);
-            logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
-
-            Iterator<DownloadJob> iterator = downloadQ.iterator();
-            boolean removed = false;
-            while (iterator.hasNext()) {
-                DownloadJob next = iterator.next();
-                if (next.getImage().getPostId().equals(image.getPostId())) {
-                    iterator.remove();
-                    appStateService.doneDownloadJob(image);
-                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
-                    removed = true;
-                    break;
-                }
-            }
-
-            if (!removed) {
-                logger.debug(String.format("Job for %s does not exist", image.getUrl()));
-            }
-
-            image.cleanup();
-        }
-    }
-
-    private void removeRunning(String postId) {
-        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
-        executionService.stop(postId);
-    }
-
-
-    public void stop(String postId) {
-        try {
-            synchronized (appStateService) {
-                if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
-                    return;
-                }
-                notPauseQ = false;
-                appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
-                List<Image> images = appStateService.getPost(postId)
-                        .getImages()
-                        .stream()
-                        .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                        .collect(Collectors.toList());
-                if (images.isEmpty()) {
-                    return;
-                }
-                logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
-                images.forEach(this::removeScheduled);
-                removeRunning(postId);
-            }
-        } finally {
-            notPauseQ = true;
-        }
-    }
-
     public int size() {
-        return downloadQ.size();
+        return downloadQ.values().stream().mapToInt(BlockingDeque::size).sum();
     }
 
-    public void stopAll() {
-        synchronized (appStateService) {
-            appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
-        }
-    }
-
-    public void restartAll() throws InterruptedException {
-        synchronized (appStateService) {
-            for (Post post : appStateService.getCurrentPosts().values()) {
-                String postId = post.getPostId();
-                restart(postId);
-            }
-        }
+    public Iterable<? extends Map.Entry<Host, BlockingDeque<DownloadJob>>> entries() {
+        return downloadQ.entrySet();
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -7,17 +7,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
+import tn.mnlr.vripper.services.ThumbnailGenerator;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
+import java.time.Duration;
 import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -27,6 +29,8 @@ public class ExecutionService {
 
     private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
 
+    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
+
     @Autowired
     private DownloadQ downloadQ;
 
@@ -36,29 +40,40 @@ public class ExecutionService {
     @Autowired
     private AppStateService appStateService;
 
-    private final AtomicInteger threadCount = new AtomicInteger();
+    @Autowired
+    private ThumbnailGenerator thumbnailGenerator;
 
-    private ExecutorService executor = Executors.newFixedThreadPool(10);
+    @Autowired
+    private AppSettingsService appSettingsService;
+    private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
+    private boolean notPauseQ = true;
+    private ExecutorService executor = Executors.newFixedThreadPool(20);
 
     private Thread executionThread;
 
+    BlockingQueue<DownloadJob> queue = new LinkedBlockingQueue<>();
+
     private RetryPolicy<Object> retryPolicy;
 
     private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
 
     private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
+    private Thread pollThread;
 
     @PostConstruct
     private void init() {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withBackoff(10, 60, ChronoUnit.SECONDS)
-                .withMaxRetries(4)
+                .withBackoff(1, 10, ChronoUnit.SECONDS)
+                .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+                .withMaxAttempts(3)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 
         executionThread = new Thread(this::start, "Executor thread");
+        pollThread = new Thread(this::poll, "Polling thread");
+        pollThread.start();
         executionThread.start();
     }
 
@@ -69,12 +84,12 @@ public class ExecutionService {
         executor.shutdown();
         appStateService.getCurrentPosts().keySet().forEach(p -> {
             logger.debug(String.format("Stopping download jobs for %s", p));
-            this.stop(p);
+            this.stopRunning(p);
         });
         executor.awaitTermination(10, TimeUnit.SECONDS);
     }
 
-    public void stop(String postId) {
+    public void stopRunning(String postId) {
         List<DownloadJob> data = running
                 .stream()
                 .filter(e -> e.getImage().getPostId().equals(postId))
@@ -84,76 +99,182 @@ public class ExecutionService {
 
         data.forEach(e -> {
             futures.get(e.getImage().getUrl()).cancel(true);
-            if(e.getImageFileData().getImageRequest() != null) {
+            if (e.getImageFileData().getImageRequest() != null) {
                 e.getImageFileData().getImageRequest().abort();
             }
             e.getImage().cleanup();
         });
     }
 
-    private boolean canRun() {
-        boolean canRun = threadCount.get() < settings.getMaxThreads();
-        if (canRun && downloadQ.isNotPauseQ()) {
-            threadCount.incrementAndGet();
+
+    public synchronized void stopAll() {
+        appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
+    }
+
+    public synchronized void restartAll() throws InterruptedException {
+        for (Post post : appStateService.getCurrentPosts().values()) {
+            String postId = post.getPostId();
+            restart(postId);
+        }
+    }
+
+    public synchronized void restart(String postId) throws InterruptedException {
+        if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
+            logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
+            return;
+        }
+        List<Image> images = appStateService.getPost(postId)
+                .getImages()
+                .stream()
+                .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                .collect(Collectors.toList());
+        if (images.isEmpty()) {
+            return;
+        }
+        appStateService.getPost(postId).setStatus(Post.Status.PENDING);
+        logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
+        for (Image image : images) {
+            downloadQ.put(image);
+        }
+    }
+
+    private synchronized void removeScheduled(Image image) {
+        image.setStatus(Image.Status.STOPPED);
+        logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
+
+        boolean removed = false;
+        main:
+        for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : downloadQ.entries()) {
+            Iterator<DownloadJob> iterator = entry.getValue().iterator();
+            while (iterator.hasNext()) {
+                DownloadJob next = iterator.next();
+                if (next.getImage().getPostId().equals(image.getPostId())) {
+                    iterator.remove();
+                    appStateService.doneDownloadJob(image);
+                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
+                    removed = true;
+                    break main;
+                }
+            }
+        }
+
+        if (!removed) {
+            logger.debug(String.format("Job for %s does not exist", image.getUrl()));
+        }
+
+        image.cleanup();
+    }
+
+    private synchronized void removeRunning(String postId) {
+        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
+        stopRunning(postId);
+    }
+
+    public synchronized void stop(String postId) {
+        try {
+            if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
+                return;
+            }
+            notPauseQ = false;
+            appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
+            List<Image> images = appStateService.getPost(postId)
+                    .getImages()
+                    .stream()
+                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                    .collect(Collectors.toList());
+            if (images.isEmpty()) {
+                return;
+            }
+            logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
+            images.forEach(this::removeScheduled);
+            removeRunning(postId);
+        } finally {
+            notPauseQ = true;
+        }
+    }
+
+    private boolean canRun(Host host) {
+        boolean canRun;
+        AtomicInteger count = threadCount.get(host);
+        if (count == null) {
+            threadCount.put(host, new AtomicInteger(0));
+        }
+        canRun = threadCount.get(host).get() < settings.getMaxThreads() && (settings.getMaxTotalThreads() == 0 || threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getMaxTotalThreads());
+        if (canRun && notPauseQ) {
+            threadCount.get(host).incrementAndGet();
             return true;
         }
         return false;
     }
 
-    public void start() {
+    public void poll() {
         while (!Thread.interrupted()) {
-            if (canRun()) {
-                DownloadJob take;
-                try {
-                    take = downloadQ.take();
-                    if (take == null) {
-                        continue;
+            try {
+                List<DownloadJob> peek = downloadQ.peek();
+                for (DownloadJob downloadJob : peek) {
+                    if (canRun(downloadJob.getImage().getHost())) {
+                        queue.offer(downloadJob);
+                        downloadQ.remove(downloadJob);
                     }
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    break;
                 }
-
-                DownloadJob finalTake = take;
-                Runnable task = () -> {
-                    running.add(finalTake);
-
-                    Failsafe.with(retryPolicy)
-                            .onFailure(e -> {
-                                if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
-                                    logger.debug("Job successfully interrupted");
-                                    return;
-                                }
-                                logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
-                                finalTake.getImage().setStatus(Image.Status.ERROR);
-                            })
-                            .onComplete(e -> {
-                                appStateService.doneDownloadJob(finalTake.getImage());
-                                logger.debug(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
-                                synchronized (threadCount) {
-                                    threadCount.decrementAndGet();
-                                    running.remove(finalTake);
-                                    futures.remove(finalTake.getImage().getUrl());
-                                    threadCount.notify();
-                                }
-                            })
-                            .get(finalTake::call);
-                };
-                logger.debug(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
-                futures.put(finalTake.getImage().getUrl(), executor.submit(task));
-            } else {
                 synchronized (threadCount) {
-                    try {
-                        threadCount.wait(2_000);
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                        break;
-                    }
+                    threadCount.wait(2_000);
                 }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
             }
         }
     }
 
+    public void start() {
+        while (!Thread.interrupted()) {
+            try {
+                push(queue.take());
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            } catch (Exception e) {
+                logger.error("Execution Service failed", e);
+                break;
+            }
+        }
+    }
+
+    private void push(DownloadJob take) {
+        Runnable task = () -> {
+            running.add(take);
+
+            Failsafe.with(retryPolicy)
+                    .onFailure(e -> {
+                        if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
+                            logger.debug("Job successfully interrupted");
+                            return;
+                        }
+                        logger.error(String.format("Failed to download %s after %d tries", take.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
+                        take.getImage().setStatus(Image.Status.ERROR);
+                    })
+                    .onComplete(e -> {
+                        appStateService.doneDownloadJob(take.getImage());
+                        logger.debug(String.format("Finished downloading %s", take.getImage().getUrl()));
+                        if (appSettingsService.isViewPhotos()) {
+                            VripperApplication.commonExecutor.submit(
+                                    () -> thumbnailGenerator.getThumbnails()
+                                            .get(new ThumbnailGenerator.CacheKey(take.getImage().getPostId(), take.getImageFileData().getFileName())));
+                        }
+                        threadCount.get(take.getImage().getHost()).decrementAndGet();
+                        running.remove(take);
+                        futures.remove(take.getImage().getUrl());
+                        synchronized (threadCount) {
+                            threadCount.notify();
+                        }
+                    })
+                    .get(take::call);
+        };
+        logger.debug(String.format("Scheduling a job for %s", take.getImage().getUrl()));
+        futures.put(take.getImage().getUrl(), executor.submit(task));
+    }
+
     public int runningCount() {
         return running.size();
     }

vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java

@@ -10,6 +10,7 @@ public class ImageFileData {
 
     private String pageUrl;
     private String imageName;
+    private String fileName;
     private String imageUrl;
     private HttpUriRequest imageRequest;
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -7,9 +7,11 @@ import lombok.Setter;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.exception.ValidationException;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
@@ -23,12 +25,18 @@ import java.util.prefs.Preferences;
 @Setter
 public class AppSettingsService {
 
+    private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
+
+    @Value("${base.dir}")
+    private String defaultDownloadPath;
+
     private static final Logger logger = LoggerFactory.getLogger(AppSettingsService.class);
 
     private Preferences prefs = Preferences.userNodeForPackage(AppSettingsService.class);
 
     private final String DOWNLOAD_PATH = "DOWNLOAD_PATH";
     private final String MAX_THREADS = "MAX_THREADS";
+    private int maxTotalThreads;
     private final String AUTO_START = "AUTO_START";
     private final String V_LOGIN = "VLOGIN";
     private final String V_USERNAME = "VUSERNAME";
@@ -40,6 +48,13 @@ public class AppSettingsService {
     private final String THREADSUBFOLDER = "THREADSUBFOLDER";
     private final String CLEAR = "CLEAR";
     private final String DARK_THEME = "DARK_THEME";
+    private final String VIEW_PHOTOS = "VIEW_PHOTOS";
+    private final String NOTIFICATION = "NOTIFICATION";
+
+    @PostConstruct
+    private void init() {
+        restore();
+    }
 
     private String downloadPath;
     private int maxThreads;
@@ -54,6 +69,8 @@ public class AppSettingsService {
     private boolean forceOrder;
     private boolean clearCompleted;
     private boolean darkTheme;
+    private boolean viewPhotos;
+    private boolean notificationEnabled;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -65,8 +82,9 @@ public class AppSettingsService {
 
     public void restore() {
 
-        downloadPath = prefs.get(DOWNLOAD_PATH, System.getProperty("user.dir"));
+        downloadPath = prefs.get(DOWNLOAD_PATH, defaultDownloadPath);
         maxThreads = prefs.getInt(MAX_THREADS, 4);
+        maxTotalThreads = prefs.getInt(MAX_TOTAL_THREADS, 0);
         autoStart = prefs.getBoolean(AUTO_START, true);
         vLogin = prefs.getBoolean(V_LOGIN, false);
         vUsername = prefs.get(V_USERNAME, "");
@@ -78,6 +96,8 @@ public class AppSettingsService {
         threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
         clearCompleted = prefs.getBoolean(CLEAR, false);
         darkTheme = prefs.getBoolean(DARK_THEME, false);
+        viewPhotos = prefs.getBoolean(VIEW_PHOTOS, false);
+        notificationEnabled = prefs.getBoolean(NOTIFICATION, false);
     }
 
     @PreDestroy
@@ -85,6 +105,7 @@ public class AppSettingsService {
 
         prefs.put(DOWNLOAD_PATH, downloadPath);
         prefs.putInt(MAX_THREADS, maxThreads);
+        prefs.putInt(MAX_TOTAL_THREADS, maxTotalThreads);
         prefs.putBoolean(AUTO_START, autoStart);
         prefs.putBoolean(V_LOGIN, vLogin);
         prefs.put(V_USERNAME, vUsername);
@@ -96,6 +117,8 @@ public class AppSettingsService {
         prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
         prefs.putBoolean(CLEAR, clearCompleted);
         prefs.putBoolean(DARK_THEME, darkTheme);
+        prefs.putBoolean(VIEW_PHOTOS, viewPhotos);
+        prefs.putBoolean(NOTIFICATION, notificationEnabled);
 
         try {
             prefs.sync();
@@ -118,6 +141,10 @@ public class AppSettingsService {
             throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
         }
 
+        if (settings.getMaxTotalThreads() < 0) {
+            throw new ValidationException(String.format("Invalid max global concurrent download settings, values must be in greater than %d", 1));
+        }
+
         if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
             throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
         }
@@ -151,6 +178,8 @@ public class AppSettingsService {
         private String downloadPath;
         @JsonProperty("maxThreads")
         private int maxThreads;
+        @JsonProperty("maxTotalThreads")
+        private int maxTotalThreads;
         @JsonProperty("autoStart")
         private boolean autoStart;
         @JsonProperty("vLogin")
@@ -171,10 +200,15 @@ public class AppSettingsService {
         private boolean threadSubLocation;
         @JsonProperty("clearCompleted")
         private boolean clearCompleted;
+        @JsonProperty("viewPhotos")
+        private boolean viewPhotos;
+        @JsonProperty("notification")
+        private boolean notification;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted) {
+        public Settings(String downloadPath, int maxThreads, int maxTotalThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted, boolean viewPhotos, boolean notification) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
+            this.maxTotalThreads = maxTotalThreads;
             this.autoStart = autoStart;
             this.vLogin = vLogin;
             this.vUsername = vUsername;
@@ -185,6 +219,8 @@ public class AppSettingsService {
             this.subLocation = subLocation;
             this.threadSubLocation = threadSubLocation;
             this.clearCompleted = clearCompleted;
+            this.viewPhotos = viewPhotos;
+            this.notification = notification;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -29,6 +29,10 @@ public class AppStateService {
 
     private Map<String, AtomicInteger> runningPosts = new ConcurrentHashMap<>();
 
+    private Map<String, QueuedVGLink> grabQueue = new ConcurrentHashMap<>();
+
+    private PublishProcessor<QueuedVGLink> liveGrabQueue = PublishProcessor.create();
+
     private PublishProcessor<Image> liveImageUpdates = PublishProcessor.create();
 
     private PublishProcessor<Post> livePostsState = PublishProcessor.create();
@@ -55,7 +59,7 @@ public class AppStateService {
         return currentPosts.get(postId);
     }
 
-    public synchronized void newDownloadJob(DownloadJob downloadJob) {
+    public void newDownloadJob(DownloadJob downloadJob) {
         String postId = downloadJob.getImage().getPostId();
         checkKeyRunningPosts(postId);
         runningPosts.get(postId).incrementAndGet();
@@ -63,7 +67,7 @@ public class AppStateService {
 
     public synchronized void postDownloadingUpdate(String postId) {
         Post post = currentPosts.get(postId);
-        if (!post.getStatus().equals(Post.Status.DOWNLOADING)) {
+        if (!post.getStatus().equals(Post.Status.DOWNLOADING) && !post.getStatus().equals(Post.Status.PARTIAL)) {
             post.setStatus(Post.Status.DOWNLOADING);
             livePostsState.onNext(post);
         }

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -26,9 +26,9 @@ public class ConnectionManager {
     private PoolingHttpClientConnectionManager pcm;
 
     private RequestConfig rc = RequestConfig.custom()
-            .setConnectionRequestTimeout(10_000)
-            .setConnectTimeout(10_000)
-            .setSocketTimeout(10_000)
+            .setConnectionRequestTimeout(5_000)
+            .setConnectTimeout(5_000)
+            .setSocketTimeout(5_000)
             .setCookieSpec(CookieSpecs.STANDARD)
             .build();
 
@@ -54,13 +54,13 @@ public class ConnectionManager {
     }
 
     public HttpGet buildHttpGet(String url) {
-        HttpGet httpGet = new HttpGet(url);
+        HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
         httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpGet;
     }
 
     public HttpPost buildHttpPost(String url) {
-        HttpPost httpPost = new HttpPost(url);
+        HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
         httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpPost;
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeed.java

@@ -13,7 +13,7 @@ public class DownloadSpeed {
     }
 
     private String formatSI(long bytes) {
-        return humanReadableByteCount(bytes, true);
+        return humanReadableByteCount(bytes, false);
     }
 
     private String humanReadableByteCount(long bytes, boolean si) {

vripper-server/src/main/java/tn/mnlr/vripper/services/PathService.java

@@ -41,13 +41,6 @@ public class PathService {
      * @return Sanitized local path string
      */
     public final String formatImageFileName(String imageName) {
-        int extensionIndex = imageName.lastIndexOf('.');
-        String fileName;
-        if (extensionIndex != -1) {
-            fileName = imageName.substring(0, extensionIndex);
-        } else {
-            fileName = imageName;
-        }
-        return sanitize(fileName);
+        return sanitize(imageName);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java

@@ -7,9 +7,10 @@ import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.EventListenerBean;
 import tn.mnlr.vripper.SpringContext;
-import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
@@ -38,8 +39,11 @@ public class PersistenceService {
     @Autowired
     private AppStateService stateService;
 
-    @Autowired
-    private VripperApplication.AppCommandRunner appCommandRunner;
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Getter
+    private String dataPath;
 
     private ObjectMapper om;
 
@@ -54,16 +58,15 @@ public class PersistenceService {
         om.addMixIn(Post.class, PostPersistanceMixin.class);
         subscription = processor
                 .onBackpressureLatest()
-                .buffer(10, TimeUnit.SECONDS)
-                .filter(e -> !e.isEmpty())
-                .map(e -> e.get(0))
+                .throttleLatest(1, TimeUnit.SECONDS)
                 .doOnNext(this::persist)
                 .subscribe();
     }
 
     @PostConstruct
     public void init() {
-        File dataFile = new File(appCommandRunner.getDataPath());
+        dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
+        File dataFile = new File(dataPath);
         if (!dataFile.exists()) {
             try {
                 if (dataFile.getParentFile().mkdirs()) {
@@ -94,12 +97,14 @@ public class PersistenceService {
         this.subscription.dispose();
         logger.info(String.format("Destroying %s", PersistenceService.class.getSimpleName()));
         logger.info("Persisting data before destroying");
-        this.persist(stateService.getCurrentPosts());
+        if (EventListenerBean.isInit()) {
+            this.persist(stateService.getCurrentPosts());
+        }
     }
 
     private void persist(Map<String, Post> currentPosts) {
 
-        try (PrintWriter out = new PrintWriter(appCommandRunner.getDataPath(), StandardCharsets.UTF_8)) {
+        try (PrintWriter out = new PrintWriter(dataPath, StandardCharsets.UTF_8)) {
             out.print(om.writeValueAsString(currentPosts));
         } catch (IOException e) {
             logger.error("Failed to persist app state", e);
@@ -113,11 +118,11 @@ public class PersistenceService {
         } catch (IOException e) {
             logger.error("Failed to read app state", e);
             long timestamp = new Date().getTime();
-            logger.warn(String.format("trying to rename old data file from %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + "." + timestamp + ".old"));
+            logger.warn(String.format("trying to rename old data file from %s to %s", dataPath, dataPath + "." + timestamp + ".old"));
             try {
-                Files.move(new File(appCommandRunner.getDataPath()).toPath(), new File(appCommandRunner.getDataPath() + "." + timestamp + ".old").toPath());
+                Files.move(new File(dataPath).toPath(), new File(dataPath + "." + timestamp + ".old").toPath());
             } catch (IOException ex) {
-                logger.error(String.format("Failed to rename %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + ".old"));
+                logger.error(String.format("Failed to rename %s to %s", dataPath, dataPath + ".old"));
                 SpringContext.close();
             }
         }
@@ -129,7 +134,7 @@ public class PersistenceService {
 
         String jsonContent = null;
         try {
-            jsonContent = String.join("", Files.readAllLines(Paths.get(appCommandRunner.getDataPath()), StandardCharsets.UTF_8));
+            jsonContent = String.join("", Files.readAllLines(Paths.get(dataPath), StandardCharsets.UTF_8));
         } catch (Exception e) {
             logger.error("data file cannot be read, previous state cannot be restored", e);
             SpringContext.close();

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -45,7 +45,7 @@ public class PostParser {
 
     }
 
-    public void addPost(String postId, String threadId) throws PostParseException {
+    public synchronized void addPost(String postId, String threadId) throws PostParseException {
 
         if (appStateService.getCurrentPosts().containsKey(postId)) {
             logger.warn(String.format("skipping %s, already loaded", postId));
@@ -77,6 +77,6 @@ public class PostParser {
     }
 
     public VRThreadParser createVRThreadParser(String threadId) {
-        return new VRThreadParser(threadId, cm, vipergirlsAuthService);
+        return new VRThreadParser(threadId, cm, vipergirlsAuthService, supportedHosts);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/QueuedVGLink.java

@@ -0,0 +1,38 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.util.Objects;
+
+@Getter
+@ToString
+public class QueuedVGLink {
+
+    private final String type = "grabQueue";
+    private final String link;
+    private final String threadId;
+    private final String postId;
+    @Setter
+    boolean removed;
+
+    public QueuedVGLink(String link, String threadId, String postId) {
+        this.link = link;
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        QueuedVGLink that = (QueuedVGLink) o;
+        return Objects.equals(link, that.link);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(link);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/ThumbnailGenerator.java

@@ -0,0 +1,161 @@
+package tn.mnlr.vripper.services;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.imgscalr.Scalr;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.util.FileSystemUtils;
+
+import javax.annotation.PostConstruct;
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Service
+public class ThumbnailGenerator {
+
+    @Getter
+    private LoadingCache<CacheKey, byte[]> thumbnails;
+
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Autowired
+    private PathService pathService;
+
+    @Getter
+    private File cacheFolder;
+    CacheLoader<CacheKey, byte[]> loader = new CacheLoader<>() {
+        @Override
+        public byte[] load(CacheKey key) throws Exception {
+            File destinationFolder = pathService.getDownloadDestinationFolder(key.getPostId());
+            if (!destinationFolder.exists() || !destinationFolder.isDirectory()) {
+                return null;
+            }
+            return Files.readAllBytes(generateThumbnail(new File(destinationFolder, key.getImgName()), key.getPostId()).toPath());
+        }
+    };
+
+    @PostConstruct
+    private void init() throws Exception {
+        cacheFolder = new File(baseDir + File.separator + ".vripper" + File.separator + "cache");
+        cacheFolder.mkdirs();
+        if (!cacheFolder.exists()) {
+            throw new Exception(String.format("%s could not be created", cacheFolder.toString()));
+        }
+        thumbnails = CacheBuilder.newBuilder()
+                .maximumSize(20000)
+                .build(loader);
+    }
+
+    public void clearCache() {
+        thumbnails.invalidateAll();
+        for (File file : Optional.ofNullable(cacheFolder.listFiles()).orElse(new File[]{})) {
+            FileSystemUtils.deleteRecursively(file);
+        }
+    }
+
+    public long cacheSize() {
+
+        final AtomicLong size = new AtomicLong(0);
+
+        try {
+            Files.walkFileTree(cacheFolder.toPath(), new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+
+                    System.out.println("skipped: " + file + " (" + exc + ")");
+                    // Skip folders that can't be traversed
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+
+                    if (exc != null)
+                        System.out.println("had trouble traversing: " + dir + " (" + exc + ")");
+                    else
+                        size.addAndGet(dir.toFile().length());
+                    // Ignore errors traversing a folder
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new AssertionError("walkFileTree will not throw IOException if the FileVisitor does not");
+        }
+
+        return size.get();
+    }
+
+    private File generateThumbnail(File inputFile, String postId) throws Exception {
+        if (!inputFile.exists()) {
+            throw new Exception(String.format("Input file %s does not exist", inputFile.toString()));
+        }
+        File postsCacheFolder = new File(cacheFolder, postId);
+        postsCacheFolder.mkdirs();
+        if (!postsCacheFolder.exists()) {
+            throw new Exception(String.format("%s could not be created", postsCacheFolder.toString()));
+        }
+
+        File thumbFile = new File(postsCacheFolder, inputFile.getName());
+        if (thumbFile.exists()) {
+            return thumbFile;
+        }
+
+        BufferedImage image = ImageIO.read(inputFile);
+        BufferedImage resize = Scalr.resize(image, 350, 350);
+        ImageIO.write(resize, "jpg", thumbFile);
+        image.flush();
+        resize.flush();
+        return thumbFile;
+    }
+
+    @Getter
+    @Setter
+    @NoArgsConstructor
+    public static class CacheKey {
+        private String postId;
+        private String imgName;
+
+        public CacheKey(String postId, String imgName) {
+            this.postId = postId;
+            this.imgName = imgName;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            CacheKey cacheKey = (CacheKey) o;
+            return Objects.equals(postId, cacheKey.postId) &&
+                    Objects.equals(imgName, cacheKey.imgName);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(postId, imgName);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VGHandler.java

@@ -0,0 +1,83 @@
+package tn.mnlr.vripper.services;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.VripperApplication;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+@Service
+public class VGHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(VGHandler.class);
+
+    @Autowired
+    private AppStateService appStateService;
+
+    @Autowired
+    private PostParser postParser;
+
+    @Getter
+    private LoadingCache<String, List<VRPostState>> cache;
+
+    @PostConstruct
+    private void init() {
+
+        CacheLoader<String, List<VRPostState>> loader = new CacheLoader<>() {
+            @Override
+            public List<VRPostState> load(String threadId) throws Exception {
+                VRThreadParser vrThreadParser = postParser.createVRThreadParser(threadId);
+                List<VRPostState> posts = new ArrayList<>();
+                vrThreadParser.parse();
+                vrThreadParser.getPostPublishProcessor()
+                        .onBackpressureBuffer()
+                        .blockingSubscribe(posts::add);
+                return posts;
+            }
+        };
+
+        cache = CacheBuilder.newBuilder()
+                .expireAfterWrite(30, TimeUnit.MINUTES)
+                .build(loader);
+    }
+
+    public void handle(List<QueuedVGLink> queuedVGLinks) throws Exception {
+        for (QueuedVGLink queuedVGLink : queuedVGLinks) {
+            if (queuedVGLink.getPostId() != null) {
+                postParser.addPost(queuedVGLink.getPostId(), queuedVGLink.getThreadId());
+            } else {
+                Callable<Void> cl = () -> {
+                    List<VRPostState> vrPostStates = cache.get(queuedVGLink.getThreadId());
+                    logger.debug(String.format("%d found for %s", vrPostStates.size(), queuedVGLink.getLink()));
+                    if (vrPostStates.size() == 1) {
+                        postParser.addPost(vrPostStates.get(0).getPostId(), vrPostStates.get(0).getThreadId());
+                        logger.debug(String.format("threadId %s, postId %s is added automatically for download", queuedVGLink.getThreadId(), queuedVGLink.getPostId()));
+                    } else {
+                        appStateService.getGrabQueue().put(queuedVGLink.getLink(), queuedVGLink);
+                        appStateService.getLiveGrabQueue().onNext(queuedVGLink);
+                    }
+                    return null;
+                };
+                VripperApplication.commonExecutor.submit(cl);
+            }
+        }
+    }
+
+    public void remove(String url) {
+        Optional.ofNullable(appStateService.getGrabQueue().remove(url)).ifPresent(e -> {
+            e.setRemoved(true);
+            appStateService.getLiveGrabQueue().onNext(e);
+        });
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostParser.java

@@ -22,6 +22,7 @@ import javax.xml.parsers.SAXParserFactory;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -39,8 +40,9 @@ class VRPostParser {
 
     private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
             .handleIf(e -> e instanceof IOException)
-            .withBackoff(5, 30, ChronoUnit.SECONDS)
-            .withMaxRetries(4)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
             .abortOn(InterruptedException.class)
             .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 
@@ -99,49 +101,22 @@ class VRPostParser {
     }
 }
 
-@Getter
-class VRPostParseState extends VRPostState {
-
-    private final String type = "postParse";
-    private String postId;
-    private int number;
-    private String title;
-    private int imageCount;
-    private String url;
-    private List<String> previews;
-
-
-    VRPostParseState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews) {
-        super(threadId);
-        this.postId = postId;
-        this.number = number;
-        this.title = title;
-        this.imageCount = imageCount;
-        this.previews = previews;
-        this.url = url;
-    }
-}
-
 class VRPostHandler extends DefaultHandler {
 
     private static final Logger logger = LoggerFactory.getLogger(VRPostHandler.class);
 
     private final List<Host> supportedHosts;
-
     private final String threadId;
     private final String postId;
+    private List<Image> images = new ArrayList<>();
+    private List<String> previews = new ArrayList<>();
     private String threadTitle;
-
+    private String postTitle;
+    private String forum;
     private int previewCounter = 0;
     private int index = 0;
-
-    private String postTitle;
     private int imageCount;
 
-    private String forum;
-
-    private List<String> previews = new ArrayList<>();
-    private List<Image> images = new ArrayList<>();
     @Getter
     private Post parsedPost;
 
@@ -172,7 +147,7 @@ class VRPostHandler extends DefaultHandler {
             case "image":
                 index++;
                 if (previewCounter++ < 4) {
-                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(thumbUrl -> previews.add(thumbUrl));
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
                 }
 
                 String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostState.java

@@ -2,11 +2,28 @@ package tn.mnlr.vripper.services;
 
 import lombok.Getter;
 
-@Getter
-public abstract class VRPostState {
-    private final String threadId;
+import java.util.List;
 
-    VRPostState(String threadId) {
+@Getter
+public class VRPostState {
+    private final String type = "postParse";
+    private final String threadId;
+    private String postId;
+    private int number;
+    private String title;
+    private int imageCount;
+    private String url;
+    private List<String> previews;
+    private String hosts;
+
+    VRPostState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
         this.threadId = threadId;
+        this.postId = postId;
+        this.number = number;
+        this.title = title;
+        this.imageCount = imageCount;
+        this.previews = previews;
+        this.url = url;
+        this.hosts = hosts;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/VRThreadParser.java

@@ -1,6 +1,6 @@
 package tn.mnlr.vripper.services;
 
-import io.reactivex.processors.PublishProcessor;
+import io.reactivex.processors.ReplayProcessor;
 import lombok.Getter;
 import net.jodah.failsafe.Failsafe;
 import net.jodah.failsafe.RetryPolicy;
@@ -15,16 +15,18 @@ import org.xml.sax.Attributes;
 import org.xml.sax.helpers.DefaultHandler;
 import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.host.Host;
 
 import javax.xml.parsers.SAXParserFactory;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.Duration;
 import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static tn.mnlr.vripper.services.PostParser.VR_API;
 
@@ -34,22 +36,25 @@ public class VRThreadParser {
 
     private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
             .handleIf(e -> e instanceof IOException)
-            .withBackoff(5, 30, ChronoUnit.SECONDS)
-            .withMaxRetries(4)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
             .abortOn(InterruptedException.class)
             .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
     private static SAXParserFactory factory = SAXParserFactory.newInstance();
     @Getter
-    private final PublishProcessor<VRPostState> postPublishProcessor = PublishProcessor.create();
+    private final ReplayProcessor<VRPostState> postPublishProcessor = ReplayProcessor.create();
     private final String threadId;
     private final ConnectionManager cm;
     private final VipergirlsAuthService vipergirlsAuthService;
+    private final List<Host> hosts;
 
 
-    VRThreadParser(String threadId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService) {
+    VRThreadParser(String threadId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> hosts) {
         this.threadId = threadId;
         this.cm = cm;
         this.vipergirlsAuthService = vipergirlsAuthService;
+        this.hosts = hosts;
     }
 
     public Void parse() throws PostParseException {
@@ -64,7 +69,7 @@ public class VRThreadParser {
             throw new PostParseException(e);
         }
 
-        VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
+        VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor, hosts);
         AtomicReference<Throwable> thr = new AtomicReference<>();
         logger.debug(String.format("Requesting %s", httpGet));
         Failsafe.with(retryPolicy)
@@ -97,23 +102,25 @@ public class VRThreadParser {
 class VRThreadHandler extends DefaultHandler {
 
     private final String threadId;
-    private final PublishProcessor<VRPostState> vrPostPublishProcessor;
+    private final ReplayProcessor<VRPostState> vrPostPublishProcessor;
+    private final List<Host> supportedHosts;
+    private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
+    private List<String> previews = new ArrayList<>();
     private String threadTitle;
     private String postId;
     private String postTitle;
     private int imageCount;
     private int postCounter;
     private int previewCounter = 0;
-    private List<String> previews = new ArrayList<>();
 
-    VRThreadHandler(String threadId, PublishProcessor<VRPostState> vrPostPublishProcessor) {
+    VRThreadHandler(String threadId, ReplayProcessor<VRPostState> vrPostPublishProcessor, List<Host> supportedHosts) {
         this.vrPostPublishProcessor = vrPostPublishProcessor;
         this.threadId = threadId;
+        this.supportedHosts = supportedHosts;
     }
 
     @Override
     public void startDocument() {
-        vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "START"));
     }
 
     @Override
@@ -130,8 +137,12 @@ class VRThreadHandler extends DefaultHandler {
                 postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
                 break;
             case "image":
+                Optional.ofNullable(attributes.getValue("main_url"))
+                        .map(String::trim)
+                        .flatMap(mainUrl -> supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
+                        .ifPresent(host -> Optional.ofNullable(hostMap.get(host)).ifPresentOrElse(AtomicInteger::incrementAndGet, () -> hostMap.put(host, new AtomicInteger(0))));
                 if (previewCounter++ < 4) {
-                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(thumbUrl -> previews.add(thumbUrl));
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
                 }
                 break;
         }
@@ -141,36 +152,25 @@ class VRThreadHandler extends DefaultHandler {
     public void endElement(String uri, String localName, String qName) {
         if ("post".equals(qName.toLowerCase())) {
             if (imageCount != 0) {
-                vrPostPublishProcessor.onNext(new VRPostParseState(
+                vrPostPublishProcessor.onNext(new VRPostState(
                         threadId,
                         postId,
                         postCounter,
                         postTitle,
                         imageCount,
                         String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
-                        previews
+                        previews,
+                        hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
                 ));
             }
             previewCounter = 0;
             previews = new ArrayList<>();
+            hostMap.clear();
         }
     }
 
     @Override
     public void endDocument() {
-        vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "END"));
         vrPostPublishProcessor.onComplete();
     }
 }
-
-@Getter
-class VRThreadParseState extends VRPostState {
-
-    private final String type = "threadParseState";
-    private final String state;
-
-    VRThreadParseState(String threadId, String state) {
-        super(threadId);
-        this.state = state;
-    }
-}

vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java

@@ -59,6 +59,11 @@ public class VipergirlsAuthService {
     @PostConstruct
     private void init() {
         context.setCookieStore(new BasicCookieStore());
+        try {
+            authenticate();
+        } catch (VripperException e) {
+            logger.error("Cannot authenticate user with ViperGirls", e);
+        }
     }
 
     @PreDestroy

vripper-server/src/main/java/tn/mnlr/vripper/web/WebSocketConfig.java

@@ -16,6 +16,6 @@ public class WebSocketConfig implements WebSocketConfigurer {
 
     @Override
     public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
-        registry.addHandler(handler, "/endpoint").setAllowedOrigins("*").withSockJS();
+        registry.addHandler(handler, "/endpoint").setAllowedOrigins("*").withSockJS().setClientLibraryUrl("../../assets/sockjs.min.js");
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/GalleryEndpoint.java

@@ -0,0 +1,202 @@
+package tn.mnlr.vripper.web.restendpoints;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.CacheControl;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import tn.mnlr.vripper.services.AppSettingsService;
+import tn.mnlr.vripper.services.PathService;
+import tn.mnlr.vripper.services.ThumbnailGenerator;
+
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+import javax.servlet.http.HttpServletResponse;
+import java.awt.*;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+@RestController
+@CrossOrigin(value = "*")
+public class GalleryEndpoint {
+
+    private static final Logger logger = LoggerFactory.getLogger(GalleryEndpoint.class);
+
+    @Autowired
+    private PathService pathService;
+
+    @Autowired
+    private ThumbnailGenerator thumbnailGenerator;
+
+    private static final String cacheControl = CacheControl.maxAge(365, TimeUnit.DAYS).getHeaderValue();
+    @Autowired
+    private AppSettingsService appSettingsService;
+
+    @ExceptionHandler(Exception.class)
+    public ResponseEntity handleException(Exception e) {
+        logger.error("Error when process request", e);
+        return ResponseEntity
+                .status(HttpStatus.INTERNAL_SERVER_ERROR)
+                .body(e.getMessage());
+    }
+
+    @GetMapping(value = "/image/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<byte[]> getImage(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        response.addHeader("Cache-Control", cacheControl);
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        return ResponseEntity.ok(Files.readAllBytes(Paths.get(destinationFolder.toPath().toString(), imgName)));
+    }
+
+    @GetMapping(value = "/image/thumb/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<byte[]> getImageThumb(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        response.addHeader("Cache-Control", cacheControl);
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        if (destinationFolder.exists() && destinationFolder.isDirectory()) {
+            return ResponseEntity.ok(thumbnailGenerator.getThumbnails().get(new ThumbnailGenerator.CacheKey(postId, imgName)));
+        }
+        return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
+    }
+
+    @GetMapping("/gallery/{postId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<List<GalleryImage>> getGallery(@PathVariable("postId") @NonNull String postId) {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        if (destinationFolder.exists() && destinationFolder.isDirectory()) {
+            return ResponseEntity.ok(
+                    Arrays.stream(Objects.requireNonNull(destinationFolder.listFiles()))
+                            .filter(f -> !f.getName().endsWith("tmp"))
+                            .filter(f -> f.getName().toLowerCase().endsWith(".jpg") || f.getName().toLowerCase().endsWith(".jpeg"))
+                            .sorted(Comparator.comparing(File::getName))
+                            .map(GalleryImage::fromFile)
+                            .filter(Objects::nonNull)
+                            .collect(Collectors.toList())
+            );
+        }
+        return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
+    }
+
+    @GetMapping("/gallery/cache")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> getCacheSize() {
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    @GetMapping("/gallery/cache/clear")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> clearCache() {
+        thumbnailGenerator.clearCache();
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    private String humanReadableByteCount(long bytes, boolean si) {
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class CacheSize {
+
+    private String size;
+
+    public CacheSize(String size) {
+        this.size = size;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class GalleryImage {
+
+    private static final CacheLoader<File, Dimension> loader = new CacheLoader<>() {
+
+        @Override
+        public Dimension load(File file) {
+            try (ImageInputStream in = ImageIO.createImageInputStream(file)) {
+                final Iterator<ImageReader> readers = ImageIO.getImageReaders(in);
+                if (readers.hasNext()) {
+                    ImageReader reader = readers.next();
+                    try {
+                        reader.setInput(in);
+                        return new Dimension(reader.getWidth(0), reader.getHeight(0));
+                    } finally {
+                        reader.dispose();
+                    }
+                } else {
+                    logger.error(String.format("No reader found for image %s", file.toString()));
+                    return null;
+                }
+            } catch (Exception e) {
+                logger.error(String.format("Failed to create image object for %s", file.toString()), e);
+                return null;
+            }
+        }
+    };
+
+    private static final LoadingCache<File, Dimension> cache = CacheBuilder.newBuilder()
+            .maximumSize(20000)
+            .build(loader);
+
+    private static final Logger logger = LoggerFactory.getLogger(GalleryImage.class);
+    private String title;
+    private String src;
+    private String msrc;
+    private double w;
+    private double h;
+
+    public GalleryImage(String title, String src, String msrc, double w, double h) {
+        this.title = title;
+        this.src = src;
+        this.msrc = msrc;
+        this.w = w;
+        this.h = h;
+    }
+
+    public static GalleryImage fromFile(File file) {
+
+        Dimension dimension = null;
+        try {
+            dimension = cache.get(file);
+        } catch (ExecutionException e) {
+            logger.error(String.format("Failed to get image dimensions for %s", file.toString()), e);
+        }
+        if (dimension == null) {
+            return null;
+        }
+        return new GalleryImage(file.getName(), file.getName(), file.getName(), dimension.getWidth(), dimension.getHeight());
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java

@@ -1,9 +1,6 @@
 package tn.mnlr.vripper.web.restendpoints;
 
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.Setter;
+import lombok.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,17 +9,17 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.PostParseException;
-import tn.mnlr.vripper.q.DownloadQ;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.AppStateService;
-import tn.mnlr.vripper.services.PathService;
-import tn.mnlr.vripper.services.PostParser;
+import tn.mnlr.vripper.q.ExecutionService;
+import tn.mnlr.vripper.services.*;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 @RestController
 @CrossOrigin(value = "*")
@@ -41,8 +38,11 @@ public class PostRestEndpoint {
     @Autowired
     private PathService pathService;
 
+    @Autowired
+    private VGHandler vgHandler;
+
     @ExceptionHandler(Exception.class)
-    public ResponseEntity handleException(Exception e) {
+    public ResponseEntity<String> handleException(Exception e) {
         logger.error("Error when process request", e);
         return ResponseEntity
                 .status(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -50,47 +50,56 @@ public class PostRestEndpoint {
     }
 
     @Autowired
-    private DownloadQ downloadQ;
+    private ExecutionService executionService;
+
 
     @Autowired
     private PostParser postParser;
 
     @PostMapping("/post")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception {
-        logger.debug(String.format("Starting to process thread: %s", url.getUrl()));
-        if (url.getUrl() == null || url.getUrl().isEmpty()) {
-            return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST);
-        } else if (!url.getUrl().startsWith("https://vipergirls.to")) {
-            return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
+    public synchronized ResponseEntity processPost(@RequestBody ThreadUrl _url) throws Exception {
+        if (_url.getUrl() == null || _url.getUrl().isEmpty()) {
+            return new ResponseEntity("Failed to process empty request", HttpStatus.BAD_REQUEST);
         }
-
-        String threadId, postId;
-        try {
-            Matcher m = VG_URL_PATTERN.matcher(url.getUrl());
-            if (m.find()) {
-                threadId = m.group(1);
-                postId = m.group(4);
-            } else {
-                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+        List<String> urls = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
+        ArrayList<QueuedVGLink> queuedVGLinks = new ArrayList<>();
+        for (String url : urls) {
+            logger.debug(String.format("Starting to process thread: %s", url));
+            if (!url.startsWith("https://vipergirls.to")) {
+                logger.error(String.format("Unsupported link %s", url));
+                continue;
             }
-        } catch (Exception e) {
-            throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+
+            String threadId, postId;
+            try {
+                Matcher m = VG_URL_PATTERN.matcher(url);
+                if (m.find()) {
+                    threadId = m.group(1);
+                    postId = m.group(4);
+                } else {
+                    throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+                }
+            } catch (Exception e) {
+                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+            }
+            queuedVGLinks.add(new QueuedVGLink(url, threadId, postId));
         }
-        return ResponseEntity.ok(new PairThreadIdPostId(threadId, postId));
+        vgHandler.handle(queuedVGLinks);
+        return ResponseEntity.ok().build();
     }
 
     @PostMapping("/post/restart")
     @ResponseStatus(value = HttpStatus.OK)
     public synchronized void restartPost(@RequestBody @NonNull List<PostId> postIds) throws Exception {
         for (PostId postId : postIds) {
-            downloadQ.restart(postId.getPostId());
+            executionService.restart(postId.getPostId());
         }
     }
 
     @PostMapping("/post/add")
     @ResponseStatus(value = HttpStatus.OK)
-    public void addPost(@RequestBody List<PostToAdd> posts) {
+    public synchronized void addPost(@RequestBody List<PostToAdd> posts) {
         for (PostToAdd post : posts) {
             VripperApplication.commonExecutor.submit(() -> {
                 try {
@@ -111,30 +120,30 @@ public class PostRestEndpoint {
 
     @PostMapping("/post/restart/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost() throws Exception {
-        downloadQ.restartAll();
+    public synchronized void restartPost() throws Exception {
+        executionService.restartAll();
     }
 
     @PostMapping("/post/stop")
     @ResponseStatus(value = HttpStatus.OK)
     public synchronized void stop(@RequestBody @NonNull List<PostId> postIds) {
         for (PostId postId : postIds) {
-            downloadQ.stop(postId.getPostId());
+            executionService.stop(postId.getPostId());
         }
     }
 
     @PostMapping("/post/stop/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stopAll() {
-        downloadQ.stopAll();
+    public synchronized void stopAll() {
+        executionService.stopAll();
     }
 
     @PostMapping("/post/remove")
     @ResponseStatus(value = HttpStatus.OK)
-    public synchronized ResponseEntity remove(@RequestBody @NonNull List<PostId> postIds) {
+    public synchronized ResponseEntity<List<RemoveResult>> remove(@RequestBody @NonNull List<PostId> postIds) {
         List<RemoveResult> result = new ArrayList<>();
         for (PostId postId : postIds) {
-            downloadQ.stop(postId.getPostId());
+            executionService.stop(postId.getPostId());
             appStateService.remove(postId.getPostId());
             result.add(new RemoveResult(postId.getPostId()));
         }
@@ -143,15 +152,27 @@ public class PostRestEndpoint {
 
     @PostMapping("/post/clear/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity clearAll() {
+    public synchronized ResponseEntity<RemoveAllResult> clearAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.clearAll()));
     }
 
     @PostMapping("/post/remove/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity removeAll() {
+    public synchronized ResponseEntity<RemoveAllResult> removeAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.removeAll()));
     }
+
+    @GetMapping("/grab/{threadId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public synchronized ResponseEntity<List<VRPostState>> grab(@PathVariable("threadId") @NonNull ThreadId threadId) throws Exception {
+        return ResponseEntity.ok(vgHandler.getCache().get(threadId.getThreadId()));
+    }
+
+    @PostMapping("/grab/remove")
+    @ResponseStatus(value = HttpStatus.OK)
+    public synchronized void grabRemove(@RequestBody @NonNull ThreadUrl threadUrl) {
+        vgHandler.remove(threadUrl.getUrl());
+    }
 }
 
 @Getter
@@ -168,23 +189,21 @@ class ThreadUrl {
 @Getter
 @Setter
 @NoArgsConstructor
-class PostId {
-    private String postId;
+class ThreadId {
+    private String threadId;
 
-    public PostId(String postId) {
-        this.postId = postId;
+    public ThreadId(String threadId) {
+        this.threadId = threadId;
     }
 }
 
 @Getter
 @Setter
 @NoArgsConstructor
-class PairThreadIdPostId {
-    private String threadId;
+class PostId {
     private String postId;
 
-    PairThreadIdPostId(String threadId, String postId) {
-        this.threadId = threadId;
+    public PostId(String postId) {
         this.postId = postId;
     }
 }
@@ -235,4 +254,32 @@ class DownloadPath {
     DownloadPath(String path) {
         this.path = path;
     }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+@ToString
+class PairThreadIdPostId {
+    private String threadId;
+    private String postId;
+
+    public PairThreadIdPostId(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        PairThreadIdPostId that = (PairThreadIdPostId) o;
+        return Objects.equals(threadId, that.threadId) &&
+                Objects.equals(postId, that.postId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(threadId, postId);
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -56,10 +56,11 @@ public class SettingsRestEndpoint {
             } catch (ValidationException e) {
                 return ResponseEntity
                         .status(HttpStatus.BAD_REQUEST)
-                        .body(new Response(e.getMessage()));
+                        .body(e.getMessage());
             }
             this.settings.setDownloadPath(settings.getDownloadPath());
             this.settings.setMaxThreads(settings.getMaxThreads());
+            this.settings.setMaxTotalThreads(settings.getMaxTotalThreads());
             this.settings.setAutoStart(settings.isAutoStart());
             this.settings.setVLogin(settings.isVLogin());
 
@@ -79,6 +80,8 @@ public class SettingsRestEndpoint {
             this.settings.setSubLocation(settings.isSubLocation());
             this.settings.setThreadSubLocation(settings.isThreadSubLocation());
             this.settings.setClearCompleted(settings.isClearCompleted());
+            this.settings.setViewPhotos(settings.isViewPhotos());
+            this.settings.setNotificationEnabled(settings.isNotification());
 
             this.settings.save();
 
@@ -94,6 +97,7 @@ public class SettingsRestEndpoint {
         return new AppSettingsService.Settings(
                 settings.getDownloadPath(),
                 settings.getMaxThreads(),
+                settings.getMaxTotalThreads(),
                 settings.isAutoStart(),
                 settings.isVLogin(),
                 settings.getVUsername(),
@@ -103,7 +107,9 @@ public class SettingsRestEndpoint {
                 settings.isForceOrder(),
                 settings.isSubLocation(),
                 settings.isThreadSubLocation(),
-                settings.isClearCompleted()
+                settings.isClearCompleted(),
+                settings.isViewPhotos(),
+                settings.isNotificationEnabled()
         );
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java

@@ -14,7 +14,6 @@ import org.springframework.web.socket.CloseStatus;
 import org.springframework.web.socket.TextMessage;
 import org.springframework.web.socket.WebSocketSession;
 import org.springframework.web.socket.handler.TextWebSocketHandler;
-import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.ui.ImageUIMixin;
@@ -56,10 +55,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private Map<String, Disposable> postsSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> postDetailsSubscriptions = new ConcurrentHashMap<>();
-    private Map<String, Disposable> vrPostParserSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> stateSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> downloadSpeedSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> userSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> grabQueueSubscriptions = new ConcurrentHashMap<>();
 
     private Map<String, Future<Void>> threadParseRequests = new ConcurrentHashMap<>();
 
@@ -71,6 +70,9 @@ public class WebSocketHandler extends TextWebSocketHandler {
         WSMessage wsMessage = om.readValue(message.getPayload(), WSMessage.class);
         WSMessage.CMD cmd = WSMessage.CMD.valueOf(wsMessage.getCmd());
         switch (cmd) {
+            case GRAB_QUEUE_SUB:
+                subscribeForGrabQueue(session);
+                break;
             case GLOBAL_STATE_SUB:
                 subscribeForGlobalState(session);
                 break;
@@ -83,9 +85,6 @@ public class WebSocketHandler extends TextWebSocketHandler {
             case POSTS_SUB:
                 subscribeForPosts(session);
                 break;
-            case THREAD_PARSING_SUB:
-                subscribeForThreadParsing(session, wsMessage.getPayload());
-                break;
             case POST_DETAILS_SUB:
                 subscribeForPostDetails(session, wsMessage.getPayload());
                 break;
@@ -93,11 +92,6 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 logger.debug(String.format("Client %s unsubscribed from post details", session.getId()));
                 Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
-            case THREAD_PARSING_UNSUB:
-                logger.debug(String.format("Client %s unsubscribed from thread parsing", session.getId()));
-                Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
-                Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
-                break;
             case POSTS_UNSUB:
                 logger.debug(String.format("Client %s unsubscribed from posts", session.getId()));
                 Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
@@ -114,6 +108,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 logger.debug(String.format("Client %s unsubscribed from user info", session.getId()));
                 Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
+            case GRAB_QUEUE_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from grab queue", session.getId()));
+                Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
         }
     }
 
@@ -220,30 +218,30 @@ public class WebSocketHandler extends TextWebSocketHandler {
         );
     }
 
-    private void subscribeForThreadParsing(WebSocketSession session, String threadId) {
+    private void subscribeForGrabQueue(WebSocketSession session) {
 
-        logger.debug(String.format("Client %s subscribed for thread parsing with threadId = %s", session.getId(), threadId));
-        if (vrPostParserSubscriptions.containsKey(session.getId())) {
-            vrPostParserSubscriptions.get(session.getId()).dispose();
+        logger.debug(String.format("Client %s subscribed for grab queue", session.getId()));
+        if (grabQueueSubscriptions.containsKey(session.getId())) {
+            grabQueueSubscriptions.get(session.getId()).dispose();
         }
 
-        if (threadParseRequests.containsKey(session.getId())) {
-            threadParseRequests.get(session.getId()).cancel(true);
+        try {
+            send(session, new TextMessage(om.writeValueAsString(appStateService.getGrabQueue().values())));
+        } catch (Exception e) {
+            logger.error("Unexpected error occurred", e);
         }
 
-        VRThreadParser vrThreadParser = postParser.createVRThreadParser(threadId);
-
-        vrPostParserSubscriptions.put(session.getId(), vrThreadParser.getPostPublishProcessor()
-                .onBackpressureBuffer()
-                .observeOn(Schedulers.io())
-                .buffer(200)
-                .filter(e -> !e.isEmpty())
-                .map(om::writeValueAsString)
-                .map(TextMessage::new)
-                .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        grabQueueSubscriptions.put(session.getId(),
+                appStateService.getLiveGrabQueue()
+                        .onBackpressureBuffer()
+                        .observeOn(Schedulers.io())
+                        .buffer(2000, TimeUnit.MILLISECONDS, 200)
+                        .filter(e -> !e.isEmpty())
+                        .map(e -> e.stream().distinct().collect(Collectors.toList()))
+                        .map(om::writeValueAsString)
+                        .map(TextMessage::new)
+                        .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
         );
-
-        threadParseRequests.put(session.getId(), VripperApplication.commonExecutor.submit(vrThreadParser::parse));
     }
 
     private void subscribeForPostDetails(WebSocketSession session, String postId) {
@@ -295,7 +293,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
         Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
         Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
         Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
-        Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
         Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
 
         logger.debug(String.format("Connection closed for client id: %s", session.getId()));
@@ -318,10 +316,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
             GLOBAL_STATE_UNSUB,
             SPEED_SUB,
             SPEED_UNSUB,
-            THREAD_PARSING_SUB,
-            THREAD_PARSING_UNSUB,
             USER_SUB,
-            USER_UNSUB
+            USER_UNSUB,
+            GRAB_QUEUE_SUB,
+            GRAB_QUEUE_UNSUB
         }
     }
 

vripper-server/src/main/resources/application.properties

@@ -6,5 +6,5 @@ server.port=${vripper.server.port:8080}
 management.endpoints.web.exposure.include=shutdown
 management.endpoint.shutdown.enabled=true
 endpoints.shutdown.enabled=true
-spring.profiles.active=portable
-#spring.profiles.active=installer
+#spring.profiles.active=portable
+spring.profiles.active=installer

vripper-ui/angular.json

@@ -27,9 +27,14 @@
               "src/assets"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "./node_modules/photoswipe/dist/photoswipe.css",
+              "./node_modules/photoswipe/dist/default-skin/default-skin.css"
             ],
-            "scripts": []
+            "scripts": [
+              "./node_modules/photoswipe/dist/photoswipe.min.js",
+              "./node_modules/photoswipe/dist/photoswipe-ui-default.min.js"
+            ]
           },
           "configurations": {
             "production": {
@@ -83,9 +88,14 @@
             "tsConfig": "src/tsconfig.spec.json",
             "karmaConfig": "src/karma.conf.js",
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "./node_modules/photoswipe/dist/photoswipe.css",
+              "./node_modules/photoswipe/dist/default-skin/default-skin.css"
+            ],
+            "scripts": [
+              "./node_modules/photoswipe/dist/photoswipe.min.js",
+              "./node_modules/photoswipe/dist/photoswipe-ui-default.min.js"
             ],
-            "scripts": [],
             "assets": [
               "src/favicon.ico",
               "src/assets"

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.3.1",
+  "version": "2.10.5",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -13,36 +13,37 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^7.2.3",
-    "@angular/cdk": "^7.3.1",
-    "@angular/common": "~7.2.0",
-    "@angular/compiler": "~7.2.0",
-    "@angular/core": "~7.2.0",
-    "@angular/flex-layout": "^7.0.0-beta.23",
-    "@angular/forms": "~7.2.0",
-    "@angular/material": "^7.3.0",
-    "@angular/platform-browser": "~7.2.0",
-    "@angular/platform-browser-dynamic": "~7.2.0",
-    "@angular/router": "~7.2.0",
-    "ag-grid-angular": "^21.2.1",
-    "ag-grid-community": "^21.2.1",
+    "@angular/animations": "^8.2.14",
+    "@angular/cdk": "^8.2.3",
+    "@angular/common": "~8.2.14",
+    "@angular/compiler": "~8.2.14",
+    "@angular/core": "~8.2.14",
+    "@angular/flex-layout": "^8.0.0-beta.27",
+    "@angular/forms": "~8.2.14",
+    "@angular/material": "^8.2.3",
+    "@angular/platform-browser": "~8.2.14",
+    "@angular/platform-browser-dynamic": "~8.2.14",
+    "@angular/router": "~8.2.14",
+    "ag-grid-angular": "^22.1.0",
+    "ag-grid-community": "^22.1.0",
     "core-js": "^2.5.4",
     "hammerjs": "^2.0.8",
-    "ngx-electron": "^2.1.1",
-    "rxjs": "~6.3.3",
+    "ngx-electron": "^2.2.0",
+    "photoswipe": "^4.1.3",
+    "rxjs": "~6.5.3",
     "tslib": "^1.9.0",
     "zone.js": "~0.8.26"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^0.13.9",
-    "@angular/cli": "~7.2.2",
-    "@angular/compiler-cli": "^7.2.15",
-    "@angular/language-service": "~7.2.0",
+    "@angular-devkit/build-angular": "^0.803.23",
+    "@angular/cli": "~8.3.20",
+    "@angular/compiler-cli": "^8.2.14",
+    "@angular/language-service": "~8.2.14",
     "@types/jasmine": "~2.8.8",
     "@types/jasminewd2": "~2.0.3",
     "@types/node": "~8.9.4",
     "codelyzer": "~4.5.0",
-    "electron": "^6.0.9",
+    "electron": "^7.1.7",
     "jasmine-core": "~2.99.1",
     "jasmine-spec-reporter": "~4.2.1",
     "karma": "^4.2.0",
@@ -53,6 +54,6 @@
     "protractor": "~5.4.0",
     "ts-node": "~7.0.0",
     "tslint": "~5.11.0",
-    "typescript": "~3.2.2"
+    "typescript": "~3.5.0"
   }
 }

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.3.1</version>
+    <version>2.10.5</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.component.html

@@ -1,17 +1,25 @@
-<div fxLayout="column" *ngIf="connecting()" class="overlay loading" fxLayoutAlign="center center">
-  <mat-spinner></mat-spinner>
-  <h2>Loading...</h2>
-</div>
-<div fxLayout="column" *ngIf="noConnectionState()" class="overlay no-connection" fxLayoutAlign="center center">
-  <mat-icon class="overlay-icon">error</mat-icon>
-  <h2>The app seems to be shutdown</h2>
-</div>
-<div [ngClass]="{ electron: electronService.isElectronApp }" fxLayout="column" id="app-container">
-  <div id="app-header" fxFlex="nogrow">
-    <app-toolbar></app-toolbar>
+<ng-container *ngIf="appState | async as state">
+  <div *ngIf="state === 'CONNECTING'" class="overlay loading" fxLayout="column" fxLayoutAlign="center center">
+    <div class="sk-chase">
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+    </div>
   </div>
-  <div id="app-body" fxFlex="grow">
-    <router-outlet></router-outlet>
+  <div *ngIf="state === 'DISCONNECTED'" class="overlay no-connection" fxLayout="column" fxLayoutAlign="center center">
+    <mat-icon class="overlay-icon">error</mat-icon>
+    <h2>Error connecting to daemon, try restarting the application</h2>
+  </div>
+  <div *ngIf="state === 'CONNECTED'" [ngClass]="{ electron: electron | async }" fxLayout="column" id="app-container">
+    <div fxFlex="nogrow" id="app-header">
+      <app-toolbar></app-toolbar>
+    </div>
+    <div fxFlex="grow" id="app-body" style="margin-bottom: 20px;">
+      <router-outlet></router-outlet>
+    </div>
     <app-status-bar style="width: 100%"></app-status-bar>
   </div>
-</div>
+</ng-container>

vripper-ui/src/app/app.component.scss

@@ -1,13 +1,76 @@
-#title {
-  user-select: none
-}
-
 .overlay-icon {
-  font-size: 128px;
-  width: 128px;
-  height: 128px;
+  font-size: 64px;
+  width: 64px;
+  height: 64px;
 }
 
-.no-connection {
-  color: #eb6060;
+.no-connection, .loading {
+  background-color: #2c3e50;
 }
+
+.overlay {
+  position: absolute;
+  display: block;
+  width: 100%;
+  height: 100%;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  z-index: 2000;
+}
+
+.sk-chase {
+  width: 60px;
+  height: 60px;
+  position: relative;
+  animation: sk-chase 2.5s infinite linear both;
+}
+
+.sk-chase-dot {
+  width: 100%;
+  height: 100%;
+  position: absolute;
+  left: 0;
+  top: 0; 
+  animation: sk-chase-dot 2.0s infinite ease-in-out both; 
+}
+
+.sk-chase-dot:before {
+  content: '';
+  display: block;
+  width: 25%;
+  height: 25%;
+  background-color: #fff;
+  border-radius: 100%;
+  animation: sk-chase-dot-before 2.0s infinite ease-in-out both; 
+}
+
+.sk-chase-dot:nth-child(1) { animation-delay: -1.1s; }
+.sk-chase-dot:nth-child(2) { animation-delay: -1.0s; }
+.sk-chase-dot:nth-child(3) { animation-delay: -0.9s; }
+.sk-chase-dot:nth-child(4) { animation-delay: -0.8s; }
+.sk-chase-dot:nth-child(5) { animation-delay: -0.7s; }
+.sk-chase-dot:nth-child(6) { animation-delay: -0.6s; }
+.sk-chase-dot:nth-child(1):before { animation-delay: -1.1s; }
+.sk-chase-dot:nth-child(2):before { animation-delay: -1.0s; }
+.sk-chase-dot:nth-child(3):before { animation-delay: -0.9s; }
+.sk-chase-dot:nth-child(4):before { animation-delay: -0.8s; }
+.sk-chase-dot:nth-child(5):before { animation-delay: -0.7s; }
+.sk-chase-dot:nth-child(6):before { animation-delay: -0.6s; }
+
+@keyframes sk-chase {
+  100% { transform: rotate(360deg); } 
+}
+
+@keyframes sk-chase-dot {
+  80%, 100% { transform: rotate(360deg); } 
+}
+
+@keyframes sk-chase-dot-before {
+  50% {
+    transform: scale(0.4); 
+  } 100%, 0% {
+    transform: scale(1.0); 
+  } 
+}

vripper-ui/src/app/app.component.ts

@@ -1,56 +1,46 @@
 import { AppService } from './app.service';
 import { ClipboardService } from './clipboard.service';
 import { ElectronService } from 'ngx-electron';
-import { Component, OnInit, OnDestroy, NgZone, AfterViewInit, Renderer2 } from '@angular/core';
+import { Component, OnDestroy, AfterViewInit, Renderer2, ChangeDetectionStrategy, NgZone } from '@angular/core';
 import { MatDialog } from '@angular/material';
-import { Subscription } from 'rxjs';
+import { Subscription, BehaviorSubject, Subject, merge } from 'rxjs';
 import { WsConnectionService, WSState } from './ws-connection.service';
-import { WsHandler } from './ws-handler';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
-  styleUrls: ['./app.component.scss']
+  styleUrls: ['./app.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
-export class AppComponent implements OnInit, OnDestroy, AfterViewInit {
+export class AppComponent implements OnDestroy, AfterViewInit {
   constructor(
-    public dialog: MatDialog,
+    private dialog: MatDialog,
     private ws: WsConnectionService,
     public electronService: ElectronService,
     private clipboardService: ClipboardService,
-    private ngZone: NgZone,
     private appService: AppService,
-    private renderer: Renderer2
-  ) {}
-
-  subscriptions: Subscription[] = [];
-  websocketHandlerPromise: Promise<WsHandler>;
-  currentState: WSState;
-  themeLoaded = false;
-
-  noConnectionState(): boolean {
-    return this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR;
+    private renderer: Renderer2,
+    private ngZone: NgZone
+  ) {
+    this.electron = new BehaviorSubject(electronService.isElectronApp);
   }
 
-  connecting(): boolean {
-    return (this.currentState === WSState.INIT || this.currentState === WSState.CONNECTING) && !this.themeLoaded ;
-  }
+  private subscriptions: Subscription[] = [];
+  appState: Subject<string> = new BehaviorSubject('CONNECTING');
+  electron: Subject<boolean>;
 
   ngAfterViewInit() {
     this.appService.renderer = this.renderer;
-  }
-
-  ngOnInit() {
-    this.currentState = WSState.INIT;
-    this.subscriptions.push(this.ws.state.subscribe(e => {
-      this.currentState = e;
-      if (this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR) {
-        this.dialog.closeAll();
-      } else if (this.currentState === WSState.OPEN) {
+    this.subscriptions.push(this.ws.state.subscribe(wsState => {
+      if (wsState === WSState.CLOSE || wsState === WSState.ERROR) {
+        setTimeout(() => this.ngZone.run(() => {
+          this.dialog.closeAll();
+          this.appState.next('DISCONNECTED');
+        }), 500);
+      } else if (wsState === WSState.OPEN) {
         this.clipboardService.init();
-        this.appService
-          .loadTheme()
-          .subscribe(() => this.ngZone.run(() => this.themeLoaded = true));
+        merge(this.appService.loadTheme(), this.appService.loadSettings())
+        .subscribe(() => this.ngZone.run(() => this.appState.next('CONNECTED')));
       }
     }));
   }

vripper-ui/src/app/app.module.ts

@@ -1,3 +1,6 @@
+import { LinkCollectorService } from './link-collector.service';
+import { ContextMenuService } from './ctxt-menu.service';
+import { PostContextMenuComponent } from './posts/post-context-menu.component';
 import { PostsDataService } from './posts-data.service';
 import { AppPreviewDirective } from './common/preview-tooltip.directive';
 import { WsConnectionService } from './ws-connection.service';
@@ -34,6 +37,10 @@ import { ScanComponent } from './scan/scan.component';
 import { StatusBarComponent } from './status-bar/status-bar.component';
 import { SelectionService } from './selection-service';
 import { ToolbarComponent } from './toolbar/toolbar.component';
+import { GrabQueueComponent } from './grab-queue/grab-queue.component';
+import { UrlGrabRendererComponent } from './grab-queue/url-renderer.component';
+import { GalleryComponent } from './gallery/gallery.component';
+import { PhotoSwipeComponent } from './photo-swipe/photo-swipe.component';
 
 @NgModule({
   declarations: [
@@ -50,17 +57,25 @@ import { ToolbarComponent } from './toolbar/toolbar.component';
     AppPreviewComponent,
     AppPreviewDirective,
     UrlRendererComponent,
+    UrlGrabRendererComponent,
     FilterComponent,
     ScanComponent,
     StatusBarComponent,
-    ToolbarComponent
+    ToolbarComponent,
+    GrabQueueComponent,
+    PostContextMenuComponent,
+    GalleryComponent,
+    PhotoSwipeComponent
   ],
   entryComponents: [
     PostDetailComponent,
     SettingsComponent,
     ConfirmDialogComponent,
     AppPreviewComponent,
-    ScanComponent
+    ScanComponent,
+    MultiPostComponent,
+    PostContextMenuComponent,
+    GalleryComponent
   ],
   imports: [
     BrowserAnimationsModule,
@@ -74,7 +89,8 @@ import { ToolbarComponent } from './toolbar/toolbar.component';
     AgGridModule.withComponents([
       PostProgressRendererComponent,
       PostDetailsProgressRendererComponent,
-      UrlRendererComponent
+      UrlRendererComponent,
+      UrlGrabRendererComponent
     ]),
     OverlayModule
   ],
@@ -85,7 +101,9 @@ import { ToolbarComponent } from './toolbar/toolbar.component';
     ServerService,
     SharedService,
     SelectionService,
-    PostsDataService
+    PostsDataService,
+    ContextMenuService,
+    LinkCollectorService
   ],
   bootstrap: [AppComponent]
 })

vripper-ui/src/app/app.service.ts

@@ -1,11 +1,14 @@
+import { Settings } from './common/settings.model';
+import { environment } from 'src/environments/environment';
 import { MatDialog } from '@angular/material';
 import { HttpClient } from '@angular/common/http';
-import { Injectable, Renderer2, NgZone } from '@angular/core';
+import { Injectable, Renderer2 } from '@angular/core';
 import { ServerService } from './server-service';
 import { tap } from 'rxjs/operators';
 import { ScanComponent } from './scan/scan.component';
-import { Observable } from 'rxjs';
+import { Observable, Subject } from 'rxjs';
 import { BreakpointState, Breakpoints, BreakpointObserver } from '@angular/cdk/layout';
+import { Title } from '@angular/platform-browser';
 
 @Injectable()
 export class AppService {
@@ -14,20 +17,32 @@ export class AppService {
     private serverService: ServerService,
     public dialog: MatDialog,
     private breakpointObserver: BreakpointObserver,
-  ) {}
+    private titleService: Title
+  ) {
+    this.titleService.setTitle('VRipper ' + environment.version);
+  }
+
+  private _darkTheme = false;
+  private _settings: Settings;
+  private _renderer: Renderer2;
+
+  get settings(): Settings {
+    return { ...this._settings };
+  }
+
+  get darkTheme(): boolean {
+    return this._darkTheme;
+  }
 
-  darkTheme = false;
-  _renderer: Renderer2;
   set renderer(renderer: Renderer2) {
     this._renderer = renderer;
   }
 
   isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
-  isScanOpen = false;
 
   updateTheme(darkTheme: boolean) {
-    this.darkTheme = darkTheme;
-    if (this.darkTheme) {
+    this._darkTheme = darkTheme;
+    if (this._darkTheme) {
       this._renderer.addClass(document.body, 'dark-theme');
       this._renderer.removeClass(document.body, 'light-theme');
     } else {
@@ -36,17 +51,27 @@ export class AppService {
     }
     this.httpClient
       .post<Theme>(this.serverService.baseUrl + '/settings/theme', {
-        darkTheme: this.darkTheme
+        darkTheme: this._darkTheme
       })
       .subscribe();
   }
 
+  updateSettings(settings: Settings) {
+    this._settings = settings;
+  }
+
   loadTheme() {
     return this.httpClient
       .get<Theme>(this.serverService.baseUrl + '/settings/theme')
       .pipe(tap(theme => this.updateTheme(theme.darkTheme)));
   }
 
+  loadSettings() {
+    return this.httpClient
+      .get<Settings>(this.serverService.baseUrl + '/settings')
+      .pipe(tap(settings => this.updateSettings(settings)));
+  }
+
   scan(url?: string) {
 
     const scanDialog = this.dialog.open(ScanComponent, {
@@ -65,11 +90,8 @@ export class AppService {
       }
     });
 
-    scanDialog.afterOpened().subscribe(() => this.isScanOpen = true);
-
     scanDialog.afterClosed().subscribe(() => {
       smallDialogSubscription.unsubscribe();
-      this.isScanOpen = false;
     });
   }
 }

vripper-ui/src/app/clipboard.service.ts

@@ -1,6 +1,6 @@
 import { Subject, Observable } from 'rxjs';
 import { ElectronService } from 'ngx-electron';
-import { Injectable, OnInit } from '@angular/core';
+import { Injectable } from '@angular/core';
 import { Clipboard } from 'electron';
 import { HttpClient } from '@angular/common/http';
 import { Settings } from './common/settings.model';

vripper-ui/src/app/common/cmd.enum.ts

@@ -5,10 +5,12 @@ export enum CMD {
     SPEED_SUB = 'SPEED_SUB',
     USER_SUB = 'USER_SUB',
     THREAD_PARSING_SUB = 'THREAD_PARSING_SUB',
+    GRAB_QUEUE_SUB = 'GRAB_QUEUE_SUB',
     POSTS_UNSUB = 'POSTS_UNSUB',
     POST_DETAILS_UNSUB = 'POST_DETAILS_UNSUB',
     GLOBAL_STATE_UNSUB = 'GLOBAL_STATE_UNSUB',
     SPEED_UNSUB = 'SPEED_UNSUB',
     USER_UNSUB = 'USER_UNSUB',
-    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB'
+    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB',
+    GRAB_QUEUE_UNSUB = 'GRAB_QUEUE_UNSUB'
 }

vripper-ui/src/app/common/preview-tooltip.component.ts

@@ -4,11 +4,10 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
 @Component({
   selector: 'app-preview',
   template: `
-  <div class="previews" style="display: inline-block; white-space: nowrap;">
+  <div class="previews" style="display: inline-block; white-space: nowrap; height: 200px; max-width: 400px">
     <ng-container *ngFor="let link of links">
-      <img style="display: inline-block; max-width:150px; min-width:150px; width: auto; height: auto;"
+      <img
         [@simpleFadeAnimation]="'in'"
-        class="mat-elevation-z8"
         [src]="link"
       />
     </ng-container>
@@ -18,7 +17,7 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
     `
       img {
         max-height: 200px;
-        max-width: 200px;
+        max-width: 400px;
         margin-left: 5px;
       }
     `

vripper-ui/src/app/common/preview-tooltip.directive.ts

@@ -1,32 +1,48 @@
 import { AppPreviewComponent } from './preview-tooltip.component';
-import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone } from '@angular/core';
+import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone, OnDestroy } from '@angular/core';
 import { OverlayRef, Overlay, OverlayPositionBuilder } from '@angular/cdk/overlay';
 import { ComponentPortal } from '@angular/cdk/portal';
 
 @Directive({ selector: '[appPreview]' })
-export class AppPreviewDirective implements OnInit {
+export class AppPreviewDirective implements OnInit, OnDestroy {
   private overlayRef: OverlayRef;
   tooltipPortal = new ComponentPortal(AppPreviewComponent);
 
   @Input() appPreview: string[];
 
-  constructor(private overlayPositionBuilder: OverlayPositionBuilder,
+  constructor(
+    private overlayPositionBuilder: OverlayPositionBuilder,
     private elementRef: ElementRef,
     private overlay: Overlay,
-    private ngZone: NgZone) {}
+    private ngZone: NgZone
+  ) {}
 
   ngOnInit() {
     const positionStrategy = this.overlayPositionBuilder
-    .flexibleConnectedTo(this.elementRef)
-    .withPush(true)
-    .withGrowAfterOpen(true)
-    .withPositions([{
-      originX: 'start',
-      originY: 'bottom',
-      overlayX: 'start',
-      overlayY: 'top',
-    }]);
-    this.overlayRef = this.overlay.create({ positionStrategy });
+      .flexibleConnectedTo(this.elementRef)
+      .withPush(true)
+      .withGrowAfterOpen(true)
+      .withPositions([
+        {
+          originX: 'center',
+          originY: 'bottom',
+          overlayX: 'center',
+          overlayY: 'top'
+        },
+        {
+          originX: 'center',
+          originY: 'top',
+          overlayX: 'center',
+          overlayY: 'bottom'
+        }
+      ]);
+    this.overlayRef = this.overlay.create({ positionStrategy, scrollStrategy: this.overlay.scrollStrategies.close() });
+  }
+
+  ngOnDestroy() {
+    this.ngZone.run(() => {
+      this.overlayRef.dispose();
+    });
   }
 
   @HostListener('mouseenter')

vripper-ui/src/app/common/settings.model.ts

@@ -1,10 +1,13 @@
 export interface Settings {
     downloadPath: string;
     maxThreads: number;
+    maxTotalThreads: number;
     autoStart: boolean;
     vLogin: boolean;
     vUsername: string;
     vPassword: string;
     vThanks: boolean;
     desktopClipboard: boolean;
-  }
+    viewPhotos: boolean;
+    notification: boolean;
+  }

vripper-ui/src/app/common/vr-post-parse.model.ts

@@ -6,10 +6,7 @@ export class VRPostParse {
     public title: string,
     public imageCount: number,
     public url: string,
-    public previews: string[]
+    public previews: string[],
+    public hosts: string
   ) {}
 }
-
-export class VRThreadParseState {
-  constructor(public threadId: string, public state: string) {}
-}

vripper-ui/src/app/ctxt-menu.service.ts

@@ -0,0 +1,68 @@
+import { PostState } from './posts/post-state.model';
+import { Injectable, ComponentRef } from '@angular/core';
+import { filter, take } from 'rxjs/operators';
+import { fromEvent, Subscription } from 'rxjs';
+import { OverlayPositionBuilder, Overlay, OverlayRef } from '@angular/cdk/overlay';
+import { ComponentPortal } from '@angular/cdk/portal';
+import { PostContextMenuComponent } from './posts/post-context-menu.component';
+
+@Injectable()
+export class ContextMenuService {
+  constructor(private overlayPositionBuilder: OverlayPositionBuilder, private overlay: Overlay) {}
+
+  postCtxtMenuPortal = new ComponentPortal(PostContextMenuComponent);
+  private postCtxtMenuOverlayRef: OverlayRef;
+  sub: Subscription;
+
+  openPostCtxtMenu(mouseEvent: MouseEvent, postState: PostState) {
+    mouseEvent.preventDefault();
+    this.closePostCtxtMenu();
+    const positionStrategy = this.overlayPositionBuilder
+      .flexibleConnectedTo({ x: mouseEvent.x, y: mouseEvent.y })
+      .withPush(true)
+      .withGrowAfterOpen(true)
+      .withPositions([
+        {
+          originX: 'start',
+          originY: 'bottom',
+          overlayX: 'start',
+          overlayY: 'top'
+        },
+        {
+          originX: 'start',
+          originY: 'top',
+          overlayX: 'start',
+          overlayY: 'bottom'
+        }
+      ]);
+    this.postCtxtMenuOverlayRef = this.overlay.create({
+      positionStrategy
+    });
+    const ref: ComponentRef<PostContextMenuComponent> = this.postCtxtMenuOverlayRef.attach(this.postCtxtMenuPortal);
+    ref.instance.postState = postState;
+    ref.instance.contextMenuService = this;
+    this.sub = fromEvent<MouseEvent>(document, 'click')
+      .pipe(
+        filter(event => {
+          const clickTarget = event.target as HTMLElement;
+          return !!this.postCtxtMenuOverlayRef && !this.postCtxtMenuOverlayRef.overlayElement.contains(clickTarget);
+        }),
+        take(1)
+      )
+      .subscribe(() => this.closePostCtxtMenu());
+  }
+
+  closePostCtxtMenu() {
+    if (this.sub != null) {
+      this.sub.unsubscribe();
+    }
+    if (this.postCtxtMenuOverlayRef) {
+      this.postCtxtMenuOverlayRef.dispose();
+      this.postCtxtMenuOverlayRef = null;
+    }
+  }
+}
+
+export interface Theme {
+  darkTheme: boolean;
+}

vripper-ui/src/app/gallery/gallery.component.html

@@ -0,0 +1,40 @@
+<div class="dialog-container" fxLayout="column" style="height: 100%;">
+  <div fxFlex="nogrow">
+    <h2 class="no-wrap" mat-dialog-title>{{ dialogData.title }}</h2>
+  </div>
+  <mat-dialog-content fxFlex="grow">
+    <div class="container">
+      <div class="masonry">
+        <ng-container *ngFor="let img of images | async; let i = index">
+          <figure class="masonry-brick">
+            <img
+              [style.opacity]="img._initialized ? 1 : 0"
+              [style.visibility]="img._initialized ? 'visible' : 'hidden'"
+              (click)="openSlideshow(i)"
+              (load)="loaded(img)"
+              [src]="img.msrc"
+            />
+          </figure>
+        </ng-container>
+
+        <app-photo-swipe #photoSwipe></app-photo-swipe>
+      </div>
+    </div>
+  </mat-dialog-content>
+  <div style="height: 5px; width: 100%; margin-top: 8px">
+    <div *ngIf="loadProgress | async as progress">
+      <mat-progress-bar
+        *ngIf="!progress.done"
+        [mode]="progress.loading ? 'determinate' : 'indeterminate'"
+        [value]="progress.progress"
+      ></mat-progress-bar>
+    </div>
+  </div>
+  <mat-dialog-actions align="end" fxFlex="nogrow">
+    <button (click)="refresh()" [disabled]="disableRefresh | async" color="primary" mat-raised-button>
+      <mat-icon>refresh</mat-icon>
+      <span>Refresh</span>
+    </button>
+    <button mat-dialog-close mat-raised-button>Close</button>
+  </mat-dialog-actions>
+</div>

vripper-ui/src/app/gallery/gallery.component.scss

@@ -0,0 +1,24 @@
+.container {
+  height: 100%;
+}
+
+.masonry {
+  width: 100%;
+}
+
+.masonry-brick {
+  height: 250px;
+  width: 250px;
+  display: inline-block;
+  cursor: pointer;
+  margin: 0 3px;
+}
+
+img {
+  opacity: 0;
+  visibility: hidden;
+  transition: opacity 0.5s ease-in, visibility 0.5s;
+  object-fit: cover;
+  height: 250px;
+  width: 250px;
+}

vripper-ui/src/app/gallery/gallery.component.ts

@@ -0,0 +1,117 @@
+import { IImage, PhotoSwipeComponent } from './../photo-swipe/photo-swipe.component';
+import { Component, OnInit, ChangeDetectionStrategy, Inject, NgZone, OnDestroy, ViewChild } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { MatDialogRef, MAT_DIALOG_DATA, MatSnackBar } from '@angular/material';
+import { ServerService } from '../server-service';
+import { PostState } from '../posts/post-state.model';
+import { BehaviorSubject, Subject } from 'rxjs';
+
+class Image extends IImage {
+  constructor(
+    public title: string,
+    public src: string,
+    public msrc: string,
+    public w: number,
+    public h: number,
+    public _initialized: boolean
+  ) {
+    super(src, msrc, w, h);
+  }
+}
+
+interface Progress {
+  loading: boolean;
+  progress: number;
+  done: boolean;
+}
+
+const initialProgressState: Progress = { loading: false, progress: 0, done: false };
+
+@Component({
+  selector: 'app-gallery',
+  templateUrl: './gallery.component.html',
+  styleUrls: ['./gallery.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class GalleryComponent implements OnInit, OnDestroy {
+  constructor(
+    public dialogRef: MatDialogRef<GalleryComponent>,
+    @Inject(MAT_DIALOG_DATA) public dialogData: PostState,
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    private _snackBar: MatSnackBar,
+    private ngZone: NgZone
+  ) {}
+
+  images: Subject<Image[]> = new BehaviorSubject([]);
+  _images: Image[] = [];
+  disableRefresh: Subject<boolean> = new BehaviorSubject(true);
+  loadProgress: Subject<Progress> = new BehaviorSubject(initialProgressState);
+  loadedImages = 0;
+
+  @ViewChild('photoSwipe', { static: true }) photoSwipe: PhotoSwipeComponent;
+
+  loaded(img: Image) {
+    img._initialized = true;
+    this.loadedImages++;
+    this.loadProgress.next({
+      loading: true,
+      progress: Math.floor((this.loadedImages / this._images.length) * 100),
+      done: this.loadedImages === this._images.length
+    });
+    if (this.loadedImages === this._images.length) {
+      this.disableRefresh.next(false);
+    }
+  }
+
+  ngOnInit() {
+    this.refresh();
+  }
+
+  ngOnDestroy() {}
+
+  openSlideshow(index: number) {
+    this.photoSwipe.openGallery(this._images, {
+      index: index,
+      getThumbBoundsFn: this.getThumbBoundsFn,
+      showHideOpacity: true,
+      preload: [1, 8]
+    });
+  }
+
+  getThumbBoundsFn(index: number) {
+    // See Options -> getThumbBoundsFn section of documentation for more info
+    const thumbnail = document.querySelectorAll('.masonry-brick img')[index],
+      pageYScroll = window.pageYOffset || document.documentElement.scrollTop,
+      rect = thumbnail.getBoundingClientRect();
+
+    return { x: rect.left, y: rect.top + pageYScroll, w: rect.width };
+  }
+
+  refresh() {
+    this.loadProgress.next(initialProgressState);
+    this.loadedImages = 0;
+    this.disableRefresh.next(true);
+    this.httpClient.get<Image[]>(this.serverService.baseUrl + '/gallery/' + this.dialogData.postId).subscribe(
+      response => {
+        this.ngZone.run(() => {
+          response.forEach(i => {
+            i.src = this.serverService.baseUrl + '/image/' + this.dialogData.postId + '/' + i.src;
+            i.msrc = this.serverService.baseUrl + '/image/thumb/' + this.dialogData.postId + '/' + i.msrc;
+            i._initialized = false;
+          });
+          this._images = response;
+          this.images.next(this._images);
+        });
+      },
+      error => {
+        this.ngZone.run(() => {
+          this.dialogRef.close();
+          this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+            duration: 5000
+          });
+        });
+      }
+    );
+  }
+}

vripper-ui/src/app/grab-queue/grab-queue.component.html

@@ -0,0 +1,2 @@
+<ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-material" style="width: 100%; height: 100%;">
+</ag-grid-angular>

vripper-ui/src/app/grab-queue/grab-queue.component.ts

@@ -0,0 +1,58 @@
+import { LinkCollectorService } from './../link-collector.service';
+import { UrlGrabRendererComponent } from './url-renderer.component';
+import { GrabQueueDataSource } from './grab-queue.datasource';
+import { Component, OnInit, NgZone, ChangeDetectionStrategy } from '@angular/core';
+import { GridOptions } from 'ag-grid-community';
+import { WsConnectionService } from '../ws-connection.service';
+import { NotificationService } from '../notification.service';
+
+@Component({
+  selector: 'app-grab-queue',
+  templateUrl: './grab-queue.component.html',
+  styleUrls: ['./grab-queue.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class GrabQueueComponent implements OnInit {
+  constructor(
+    private wsConnection: WsConnectionService,
+    private zone: NgZone,
+    private linkCollectorService: LinkCollectorService,
+    private notificationService: NotificationService) {
+    this.gridOptions = <GridOptions>{
+      columnDefs: [
+        {
+          headerName: 'Url',
+          field: 'link',
+          sortable: true,
+          sort: 'asc',
+          suppressMovable: true,
+          cellRenderer: 'urlCellRenderer'
+        }
+      ],
+      rowHeight: 48,
+      animateRows: true,
+      rowData: [],
+      frameworkComponents: {
+        urlCellRenderer: UrlGrabRendererComponent
+      },
+      overlayLoadingTemplate: '<span></span>',
+      overlayNoRowsTemplate: '<span></span>',
+      getRowNodeId: data => data['link'],
+      onGridReady: () => {
+        this.gridOptions.api.sizeColumnsToFit();
+        this.dataSource = new GrabQueueDataSource(this.wsConnection, this.gridOptions, this.zone, this.notificationService);
+        this.dataSource.connect();
+      },
+      onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
+      onRowDataUpdated: () => {
+        this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount());
+        this.gridOptions.api.sizeColumnsToFit();
+      }
+    };
+  }
+
+  gridOptions: GridOptions;
+  dataSource: GrabQueueDataSource;
+
+  ngOnInit() {}
+}

vripper-ui/src/app/grab-queue/grab-queue.datasource.ts

@@ -0,0 +1,69 @@
+import { GrabQueueState } from './grab-queue.model';
+import { CMD } from './../common/cmd.enum';
+import { WSMessage } from './../common/ws-message.model';
+import { Subscription } from 'rxjs';
+import { WsConnectionService } from '../ws-connection.service';
+import { NotificationService } from '../notification.service';
+import { GridOptions } from 'ag-grid-community';
+import { WsHandler } from '../ws-handler';
+import { NgZone } from '@angular/core';
+
+export class GrabQueueDataSource {
+  constructor(
+    private wsConnectionService: WsConnectionService,
+    private gridOptions: GridOptions,
+    private zone: NgZone,
+    private notificationService: NotificationService
+  ) {
+    this.websocketHandlerPromise = this.wsConnectionService.getConnection();
+  }
+
+  websocketHandlerPromise: Promise<WsHandler>;
+  subscriptions: Subscription[] = [];
+
+  connect() {
+    this.websocketHandlerPromise.then((handler: WsHandler) => {
+      console.log('Connecting to grab queue datasource');
+      this.subscriptions.push(
+        handler.subscribeForGrabQueue((e: GrabQueueState[]) => {
+          this.zone.run(() => {
+            const toAdd = [];
+            const toUpdate = [];
+            const toRemove = [];
+            e.forEach(v => {
+              if (v.removed) {
+                if (this.gridOptions.api.getRowNode(v.link) != null) {
+                  toRemove.push(this.gridOptions.api.getRowNode(v.link).data);
+                }
+                return;
+              }
+
+              if (this.gridOptions.api.getRowNode(v.link) == null) {
+                toAdd.push(v);
+              } else {
+                toUpdate.push(v);
+              }
+            });
+            this.gridOptions.api.updateRowData({ update: toUpdate, add: toAdd, remove: toRemove });
+            const count = this.gridOptions.api.getDisplayedRowCount();
+            if (count > 0 && toAdd.length > 0) {
+              this.notificationService.notifyFromGrabQueue(
+                'Link Collector',
+                `You have ${count} ${count > 1 ? 'threads' : 'thread'} waiting in the link collector`
+              );
+            }
+          });
+        })
+      );
+      handler.send(new WSMessage(CMD.GRAB_QUEUE_SUB.toString()));
+    });
+  }
+
+  disconnect() {
+    console.log('Disconnecting from grab queue datasource');
+    this.subscriptions.forEach(e => e.unsubscribe());
+    this.websocketHandlerPromise.then((handler: WsHandler) => {
+      handler.send(new WSMessage(CMD.GRAB_QUEUE_UNSUB.toString()));
+    });
+  }
+}

vripper-ui/src/app/grab-queue/grab-queue.model.ts

@@ -0,0 +1,9 @@
+export class GrabQueueState {
+  constructor(
+    public type: string,
+    public link: string,
+    public threadId: string,
+    public postId: string,
+    public removed: boolean
+  ) {}
+}

vripper-ui/src/app/grab-queue/url-renderer.component.ts

@@ -0,0 +1,98 @@
+import { ElectronService } from 'ngx-electron';
+import { MultiPostComponent } from './../multi-post/multi-post.component';
+import { OnInit, OnDestroy, Component, ChangeDetectionStrategy } from '@angular/core';
+import { AgRendererComponent } from 'ag-grid-angular';
+import { ICellRendererParams } from 'ag-grid-community';
+import { GrabQueueState } from './grab-queue.model';
+import { MatDialog, MatSnackBar } from '@angular/material';
+import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout';
+import { Observable } from 'rxjs';
+import { ServerService } from '../server-service';
+import { HttpClient } from '@angular/common/http';
+
+@Component({
+  selector: 'app-url-cell-grab',
+  template: `
+    <div fxLayout="row" fxLayoutAlign="space-between center">
+      <span style="white-space: nowrap; overflow: hidden; text-overflow: ellipsis;"
+        ><a (click)="goTo()" href="javascript:void(0)">{{ grabQueue.link }}</a></span
+      >
+      <span>
+        <button style="margin-right: 5px" (click)="grab()" color="primary" mat-mini-fab>
+          <mat-icon>get_app</mat-icon>
+        </button>
+        <button (click)="remove()" color="primary" mat-mini-fab><mat-icon>delete</mat-icon></button>
+      </span>
+    </div>
+  `,
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class UrlGrabRendererComponent implements OnInit, OnDestroy, AgRendererComponent {
+  constructor(
+    public dialog: MatDialog,
+    private breakpointObserver: BreakpointObserver,
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    private _snackBar: MatSnackBar,
+    private electronService: ElectronService
+  ) {}
+
+  grabQueue: GrabQueueState;
+  params: ICellRendererParams;
+  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
+
+  ngOnInit(): void {}
+
+  ngOnDestroy(): void {}
+
+  agInit(params: ICellRendererParams): void {
+    this.grabQueue = params.data;
+  }
+
+  refresh(params: ICellRendererParams): boolean {
+    return false;
+  }
+
+  grab() {
+    const dialogRef = this.dialog.open(MultiPostComponent, {
+      width: '90%',
+      height: '90%',
+      maxWidth: '100vw',
+      maxHeight: '100vh',
+      data: this.grabQueue
+    });
+
+    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
+      if (result.matches) {
+        dialogRef.updateSize('100%', '100%');
+      } else {
+        dialogRef.updateSize('90%', '90%');
+      }
+    });
+
+    dialogRef.afterClosed().subscribe(() => {
+      if (smallDialogSubscription != null) {
+        smallDialogSubscription.unsubscribe();
+      }
+    });
+  }
+
+  goTo() {
+    if (this.electronService.isElectronApp) {
+      this.electronService.shell.openExternal(this.grabQueue.link);
+    } else {
+      window.open(this.grabQueue.link, '_blank');
+    }
+  }
+
+  remove() {
+    this.httpClient.post(this.serverService.baseUrl + '/grab/remove', { url: this.grabQueue.link }).subscribe(
+      () => {},
+      error => {
+        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+          duration: 5000
+        });
+      }
+    );
+  }
+}

vripper-ui/src/app/home/home.component.html

@@ -1,3 +1,14 @@
-<div style="height: calc(100% - 20px)">
-  <app-posts style="width: 100%; height: 100%;"></app-posts>
-</div>
+<mat-tab-group dynamicHeight style="height: 100%; margin-bottom: 20px; overflow: hidden;">
+  <mat-tab label="Downloads">
+    <app-posts style="width: 100%; height: 100%;"></app-posts>
+  </mat-tab>
+  <mat-tab>
+    <ng-template mat-tab-label>
+      <span style="margin-right: 5px;">Link Collector</span>
+      <ng-container *ngIf="linkCollectorService.count | async as count">
+        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count">link</mat-icon>
+      </ng-container>
+    </ng-template>
+    <app-grab-queue style="width: 100%; height: 100%;"></app-grab-queue>
+  </mat-tab>
+</mat-tab-group>

vripper-ui/src/app/home/home.component.scss

@@ -0,0 +1,49 @@
+table {
+  width: 100%;
+}
+
+:host ::ng-deep .mat-tab-body-wrapper {
+  height: 100%;
+}
+
+.app-custom-plain-container-with-desc {
+  align-items: center;
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  flex-wrap: wrap;
+
+  figure {
+    margin: 0;
+    position: relative;
+
+    img {
+      max-width: 100%;
+      height: auto;
+      cursor: pointer;
+    }
+
+    figcaption {
+      background: rgba(0, 0, 0, .5);
+      color: #fff;
+      font-size: 85%;
+      padding: 5px;
+      position: absolute;
+      bottom: 3px;
+      left: 0;
+      right: 0;
+    }
+  }
+
+  .description {
+    font-weight: 400;
+    text-align: center;
+  }
+
+  .app-custom-image-with-desc {
+    height: auto;
+    margin-right: 2px;
+    width: 200px;
+    align-self: start;
+  }
+}

vripper-ui/src/app/home/home.component.ts

@@ -1,8 +1,10 @@
-import { AppService } from './../app.service';
+import { LinkCollectorService } from './../link-collector.service';
+import { ServerService } from './../server-service';
 import { ElectronService } from 'ngx-electron';
 import { ClipboardService } from './../clipboard.service';
 import { Component, OnInit, OnDestroy, NgZone, ChangeDetectionStrategy } from '@angular/core';
 import { MatDialog, MatSnackBar } from '@angular/material';
+import { HttpClient } from '@angular/common/http';
 
 @Component({
   selector: 'app-home',
@@ -11,30 +13,37 @@ import { MatDialog, MatSnackBar } from '@angular/material';
   changeDetection: ChangeDetectionStrategy.OnPush
 })
 export class HomeComponent implements OnInit, OnDestroy {
-
   constructor(
     private clipboardService: ClipboardService,
     public dialog: MatDialog,
     public electronService: ElectronService,
-    private appService: AppService,
+    private httpClient: HttpClient,
+    private serverService: ServerService,
     private _snackBar: MatSnackBar,
-    private ngZone: NgZone
+    private ngZone: NgZone,
+    public linkCollectorService: LinkCollectorService
   ) {}
 
   ngOnInit() {
     this.clipboardService.links.subscribe(e => {
       this.ngZone.run(() => {
-        if (this.appService.isScanOpen) {
-          this._snackBar.open(e + ' was not processed, the app is busy parsing another thread', null, {
-            duration: 5000
-          });
-          return;
-        }
-        this.appService.scan(e);
+        this.httpClient
+          .post<{ threadId: string; postId: string }>(this.serverService.baseUrl + '/post', { url: e })
+          .subscribe(
+            response => {
+              this._snackBar.open('Clipboard successfully scanned', null, {
+                duration: 5000
+              });
+            },
+            error => {
+              this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+                duration: 5000
+              });
+            }
+          );
       });
     });
   }
 
-  ngOnDestroy() {
-  }
+  ngOnDestroy() {}
 }

vripper-ui/src/app/link-collector.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@angular/core';
+import { Subject, Observable, BehaviorSubject } from 'rxjs';
+
+@Injectable()
+export class LinkCollectorService {
+
+    private _count$: Subject<number> = new BehaviorSubject(0);
+
+    get count(): Observable<number> {
+        return this._count$.asObservable();
+    }
+
+    setCount(count: number) {
+        this._count$.next(count);
+    }
+}

vripper-ui/src/app/material.module.ts

@@ -7,7 +7,6 @@ import {
     MatIconModule,
     MatFormFieldModule,
     MatInputModule,
-    MatProgressSpinnerModule,
     MatMenuModule,
     MatDialogModule,
     MatCardModule,
@@ -16,7 +15,10 @@ import {
     MatSnackBar,
     MatTabsModule,
     MatDividerModule,
-    MatChipsModule
+    MatChipsModule,
+    MatListModule,
+    MatProgressSpinnerModule,
+    MatBadgeModule
 } from '@angular/material';
 
 @NgModule({
@@ -28,7 +30,6 @@ import {
         MatIconModule,
         MatInputModule,
         MatFormFieldModule,
-        MatProgressSpinnerModule,
         MatMenuModule,
         MatDialogModule,
         MatCardModule,
@@ -36,7 +37,10 @@ import {
         MatSnackBarModule,
         MatTabsModule,
         MatDividerModule,
-        MatChipsModule
+        MatChipsModule,
+        MatListModule,
+        MatProgressSpinnerModule,
+        MatBadgeModule
     ],
     providers: [MatSnackBar]
 })

vripper-ui/src/app/multi-post/multi-post.component.html

@@ -1,14 +1,27 @@
-<div fxLayout="column" style="height: 100%;">
+<div class="dialog-container" fxLayout="column" style="height: 100%;">
   <div fxFlex="nogrow">
-    <form autocomplete="off" fxLayout="row" fxLayoutAlign="left center" fxLayoutGap="20px">
-      <mat-form-field fxFlex="1 1 50%">
-        <input (ngModelChange)="search($event)" matInput name="search" ngModel placeholder="Search"/>
-      </mat-form-field>
-    </form>
-  </div>
-  <ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-material" style="width: 100%; height: 100%;">
-  </ag-grid-angular>
-  <div style="height: 4px; width: 100%; padding: 5px 0 0">
-    <mat-progress-bar *ngIf="loading | async" mode="indeterminate"></mat-progress-bar>
+    <h2 class="no-wrap" mat-dialog-title>Select posts</h2>
   </div>
+  <mat-dialog-content fxFlex="grow" fxLayout="column">
+    <div fxLayout="column" style="height: 100%;">
+      <div fxFlex="nogrow">
+        <form autocomplete="off" fxLayout="row" fxLayoutAlign="left center" fxLayoutGap="20px">
+          <mat-form-field fxFlex="1 1 50%">
+            <input (ngModelChange)="search($event)" matInput name="search" ngModel placeholder="Search"/>
+          </mat-form-field>
+        </form>
+      </div>
+      <ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-material" style="width: 100%; height: 100%;">
+      </ag-grid-angular>
+      <div style="height: 4px; width: 100%; padding: 5px 0 0">
+        <mat-progress-bar *ngIf="(loading | async)" mode="indeterminate"></mat-progress-bar>
+      </div>
+    </div>
+  </mat-dialog-content>
+  <mat-dialog-actions align="end" fxFlex="nogrow">
+    <button (click)="onNoClick()" mat-raised-button>Close</button>
+    <button (click)="submit()" [disabled]="(selectedRowsCount | async) === 0" color="primary" mat-raised-button>
+      Download
+    </button>
+  </mat-dialog-actions>
 </div>

vripper-ui/src/app/multi-post/multi-post.component.ts

@@ -1,15 +1,23 @@
-import { VRPostParse, VRThreadParseState } from './../common/vr-post-parse.model';
-import { Component, OnInit, NgZone, OnDestroy, Input, Output, EventEmitter, ChangeDetectionStrategy, AfterViewInit } from '@angular/core';
-import { MatSnackBar } from '@angular/material';
-import { GridOptions } from 'ag-grid-community';
+import { finalize } from 'rxjs/operators';
+import { VRPostParse } from './../common/vr-post-parse.model';
+import {
+  Component,
+  OnInit,
+  NgZone,
+  OnDestroy,
+  EventEmitter,
+  ChangeDetectionStrategy,
+  AfterViewInit,
+  Inject
+} from '@angular/core';
+import { MatSnackBar, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material';
+import { GridOptions, RowNode } from 'ag-grid-community';
 import { UrlRendererComponent } from './url-renderer.component';
 import { WsHandler } from '../ws-handler';
-import { Subscription } from 'rxjs';
-import { WSMessage } from '../common/ws-message.model';
-import { CMD } from '../common/cmd.enum';
-import { WsConnectionService } from '../ws-connection.service';
+import { Subscription, BehaviorSubject, Subject } from 'rxjs';
 import { HttpClient } from '@angular/common/http';
 import { ServerService } from '../server-service';
+import { GrabQueueState } from '../grab-queue/grab-queue.model';
 
 @Component({
   selector: 'app-multi-post',
@@ -18,13 +26,6 @@ import { ServerService } from '../server-service';
   changeDetection: ChangeDetectionStrategy.OnPush
 })
 export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
-
-  @Input()
-  threadId: string;
-
-  @Output()
-  done: EventEmitter<boolean> = new EventEmitter();
-
   gridOptions: GridOptions;
   websocketHandlerPromise: Promise<WsHandler>;
   subscription: Subscription;
@@ -33,12 +34,13 @@ export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
   constructor(
     private ngZone: NgZone,
     private _snackBar: MatSnackBar,
-    private wsConnectionService: WsConnectionService,
     private httpClient: HttpClient,
-    private serverService: ServerService
-  ) {
-    this.websocketHandlerPromise = this.wsConnectionService.getConnection();
-  }
+    private serverService: ServerService,
+    public dialogRef: MatDialogRef<MultiPostComponent>,
+    @Inject(MAT_DIALOG_DATA) public dialogData: GrabQueueState
+  ) {}
+
+  selectedRowsCount: Subject<number> = new BehaviorSubject(0);
 
   ngAfterViewInit(): void {
     this.loading.emit(true);
@@ -52,6 +54,7 @@ export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
           field: 'number',
           cellClass: 'no-padding',
           maxWidth: 100,
+          suppressMovable: true,
           checkboxSelection: true,
           headerCheckboxSelection: true,
           headerCheckboxSelectionFilteredOnly: false,
@@ -60,13 +63,21 @@ export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
         {
           headerName: 'Title',
           field: 'title',
+          suppressMovable: true,
           cellClass: 'no-padding'
         },
         {
           headerName: 'URL',
           field: 'url',
           cellClass: 'no-padding',
+          suppressMovable: true,
           cellRenderer: 'urlCellRenderer'
+        },
+        {
+          headerName: 'Hosts',
+          field: 'hosts',
+          suppressMovable: true,
+          cellClass: 'no-padding'
         }
       ],
       defaultColDef: {
@@ -86,80 +97,66 @@ export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
       getRowNodeId: data => data['url'],
       onGridReady: () => {
         this.gridOptions.api.sizeColumnsToFit();
-        this.connect();
+        this.grab();
       },
       onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
-      onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit()
+      onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit(),
+      onSelectionChanged: () => this.onSelectionChange(this.gridOptions.api.getSelectedNodes())
     };
   }
 
-  ngOnDestroy() {
-    this.unsubscribe();
+  onSelectionChange(data: RowNode[]) {
+    this.selectedRowsCount.next(data.length);
   }
 
+  ngOnDestroy() {}
+
   search(event) {
     this.gridOptions.api.setQuickFilter(event);
   }
 
-  private unsubscribe() {
-    if (this.subscription != null) {
-      this.subscription.unsubscribe();
-    }
-    this.websocketHandlerPromise.then((handler: WsHandler) => {
-      handler.send(new WSMessage(CMD.THREAD_PARSING_UNSUB.toString()));
-    });
-  }
-
-  connect() {
-    this.websocketHandlerPromise.then((handler: WsHandler) => {
-      console.log('Connecting to thread parsing');
-      this.subscription = handler.subscribeForThreadParsing(
-        (states: Array<VRThreadParseState>, data: Array<VRPostParse>) => {
-          if (data[0].threadId === this.threadId) {
-            this.gridOptions.api.updateRowData({ add: data });
-          }
-
-          if (states.length > 0 && states[states.length - 1].state === 'END' && states[0].threadId === this.threadId) {
-            this.ngZone.run(() => {
-              this.loading.emit(false);
-              if (this.gridOptions.api.getDisplayedRowCount() === 1) {
-                this.addPosts(
-                  [this.gridOptions.api.getDisplayedRowAtIndex(0).data].map(e => ({
-                    threadId: e.threadId,
-                    postId: e.postId
-                  }))
-                );
-                this.done.emit(true);
-              }
-            });
-          }
-        }
-      );
-      handler.send(new WSMessage(CMD.THREAD_PARSING_SUB.toString(), this.threadId));
-    });
-  }
-
   submit() {
+    this.loading.emit(true);
     const data = (<VRPostParse[]>this.gridOptions.api.getSelectedRows()).map(e => ({
       postId: e.postId,
-      threadId: this.threadId
+      threadId: this.dialogData.threadId
     }));
-    this.addPosts(data);
+
+    this.httpClient
+      .post(this.serverService.baseUrl + '/post/add', data)
+      .pipe(
+        finalize(() => {
+          this.dialogRef.close();
+          this.loading.emit(false);
+        })
+      )
+      .subscribe(
+        () => {},
+        error => {
+          this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+            duration: 5000
+          });
+        }
+      );
   }
 
-  addPosts(data: { postId: string; threadId: string }[]) {
-    this.httpClient.post(this.serverService.baseUrl + '/post/add', data).subscribe(
-      () => {
-        this._snackBar.open('Adding posts to queue', null, {
-          duration: 5000
-        });
-        this.done.emit();
-      },
-      error => {
-        this._snackBar.open(error.error, null, {
-          duration: 5000
-        });
-      }
-    );
+  grab() {
+    this.httpClient
+      .get<VRPostParse[]>(this.serverService.baseUrl + '/grab/' + this.dialogData.threadId)
+      .pipe(finalize(() => this.loading.emit(false)))
+      .subscribe(
+        data => {
+          this.gridOptions.api.updateRowData({ add: data });
+        },
+        error => {
+          this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+            duration: 5000
+          });
+        }
+      );
+  }
+
+  onNoClick() {
+    this.dialogRef.close();
   }
 }

vripper-ui/src/app/notification.service.ts

@@ -0,0 +1,26 @@
+import { Injectable } from '@angular/core';
+import { AppService } from './app.service';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class NotificationService {
+  constructor(private appService: AppService) {}
+
+  notifyFromGrabQueue(title: string, body: string) {
+    if (!this.appService.settings.notification) {
+      return;
+    }
+    if (!('Notification' in window)) {
+      return;
+    } else if (Notification.permission === 'granted') {
+      const notification = new Notification(title, { body: body, icon: 'assets/icon.png' });
+    } else if (Notification.permission !== 'denied') {
+      Notification.requestPermission().then(function(permission) {
+        if (permission === 'granted') {
+          const notification = new Notification(title, { body: body, icon: 'assets/icon.png' });
+        }
+      });
+    }
+  }
+}

vripper-ui/src/app/photo-swipe/photo-swipe.component.html

@@ -0,0 +1,51 @@
+<!-- Root element of PhotoSwipe. Must have class pswp. -->
+<div #photoSwipe aria-hidden="true" class="pswp" role="dialog" tabindex="-1">
+  <!-- Background of PhotoSwipe.
+   It's a separate element as animating opacity is faster than rgba(). -->
+  <div class="pswp__bg"></div>
+
+  <!-- Slides wrapper with overflow:hidden. -->
+  <div class="pswp__scroll-wrap">
+    <!-- Container that holds slides.
+      PhotoSwipe keeps only 3 of them in the DOM to save memory.
+      Don't modify these 3 pswp__item elements, data is added later on. -->
+    <div class="pswp__container">
+      <div class="pswp__item"></div>
+      <div class="pswp__item"></div>
+      <div class="pswp__item"></div>
+    </div>
+
+    <!-- Default (PhotoSwipeUI_Default) interface on top of sliding area. Can be changed. -->
+    <div class="pswp__ui pswp__ui--hidden">
+      <div class="pswp__top-bar">
+        <!--  Controls are self-explanatory. Order can be changed. -->
+
+        <div class="pswp__counter"></div>
+
+        <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
+
+        <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
+
+        <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
+
+        <!-- Preloader demo https://codepen.io/dimsemenov/pen/yyBWoR -->
+        <!-- element will get class pswp__preloader--active when preloader is running -->
+        <div class="pswp__preloader">
+          <div class="pswp__preloader__icn">
+            <div class="pswp__preloader__cut">
+              <div class="pswp__preloader__donut"></div>
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)"></button>
+
+      <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)"></button>
+
+      <div class="pswp__caption">
+        <div class="pswp__caption__center"></div>
+      </div>
+    </div>
+  </div>
+</div>

vripper-ui/src/app/photo-swipe/photo-swipe.component.ts

@@ -0,0 +1,23 @@
+import { Component, ViewChild, ElementRef, Input } from '@angular/core';
+
+import PhotoSwipe from 'photoswipe';
+import PhotoSwipeUI_Default from 'photoswipe/dist/photoswipe-ui-default';
+
+export class IImage {
+  constructor(public src: string, public msrc: string, public w: number, public h: number) {}
+}
+
+@Component({
+  selector: 'app-photo-swipe',
+  templateUrl: './photo-swipe.component.html',
+  styleUrls: ['./photo-swipe.component.css']
+})
+export class PhotoSwipeComponent {
+  @ViewChild('photoSwipe', { static: true }) photoSwipe: ElementRef;
+
+  openGallery(images: IImage[], options?: any) {
+    options = { ...options, history: false };
+    const gallery = new PhotoSwipe(this.photoSwipe.nativeElement, PhotoSwipeUI_Default, images, options);
+    gallery.init();
+  }
+}

vripper-ui/src/app/post-detail/post-detail.component.ts

@@ -22,10 +22,20 @@ export class PostDetailComponent implements OnInit, OnDestroy {
 
     this.gridOptions = <GridOptions> {
       columnDefs: [
+        {
+          headerName: '#',
+          field: 'index',
+          sortable: true,
+          sort: 'asc',
+          suppressMovable: true,
+          cellClass: 'col-white',
+          width: 10
+        },
         {
           headerName: 'URL',
           field: 'url',
           sortable: true,
+          suppressMovable: true,
           cellRenderer: 'progressCellRenderer',
           cellClass: 'no-padding'
         }

vripper-ui/src/app/post-detail/post-details-progress.component.html

@@ -14,7 +14,8 @@
         style="background-color: transparent; width: 100%; height: 100%;"
       >
         <div class="progress-bar" style="position: absolute; width: 100%; top: 37px; padding: 0 20px">
-          <mat-progress-bar [value]="details.progress" class="example-margin" color="primary" mode="determinate">
+          <mat-progress-bar *ngIf="(loaded | async)" [value]="details.progress" class="example-margin" color="primary"
+                            mode="determinate">
           </mat-progress-bar>
         </div>
         <span fxLayout="row">

vripper-ui/src/app/post-detail/post-details-progress.component.scss-theme.scss

@@ -13,7 +13,7 @@
   }
 
   app-details-cell .error {
-    background-color: mat-color($red, 50);
+    background-color: mat-color($red, 100);
     & .progress-bar .mat-progress-bar-fill::after {
       background-color: mat-color($red, 300);
     }

vripper-ui/src/app/post-detail/post-details-progress.component.ts

@@ -9,7 +9,7 @@ import {
   AfterViewInit
 } from '@angular/core';
 import { AgRendererComponent } from 'ag-grid-angular';
-import { Subscription } from 'rxjs';
+import { Subscription, Subject, BehaviorSubject } from 'rxjs';
 import { PostDetails } from './post-details.model';
 import { WsHandler } from '../ws-handler';
 import { ICellRendererParams } from 'ag-grid-community';
@@ -34,6 +34,8 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
   subscription: Subscription;
   postDetails$: EventEmitter<PostDetails> = new EventEmitter();
   private postDetails: PostDetails;
+  loaded: Subject<boolean> = new BehaviorSubject(false);
+  loading;
 
   trunc(value: number): number {
     return Math.trunc(value);
@@ -64,12 +66,14 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
 
   ngAfterViewInit(): void {
     this.postDetails$.emit(this.postDetails);
+    this.loading = setTimeout(() => this.loaded.next(true), 100);
   }
 
   ngOnDestroy(): void {
     if (this.subscription != null) {
       this.subscription.unsubscribe();
     }
+    clearTimeout(this.loading);
   }
 
   agInit(params: ICellRendererParams): void {

vripper-ui/src/app/post-detail/post-details.model.ts

@@ -1,10 +1,10 @@
 export class PostDetails {
-
-    constructor(public postId: string, public postName: string, public url: string, public progress: number, public status: string) {
-      this.postId = postId;
-      this.postName = postName;
-      this.url = url;
-      this.progress = progress;
-      this.status = status;
-    }
-  }
+  constructor(
+    public postId: string,
+    public postName: string,
+    public url: string,
+    public progress: number,
+    public status: string,
+    public index: number
+  ) {}
+}

vripper-ui/src/app/posts/post-context-menu.component.ts

@@ -0,0 +1,249 @@
+import { AppService } from './../app.service';
+import { GalleryComponent } from './../gallery/gallery.component';
+import { ServerService } from './../server-service';
+import { HttpClient } from '@angular/common/http';
+import { ElectronService } from 'ngx-electron';
+import { Component, ChangeDetectionStrategy, NgZone, OnDestroy } from '@angular/core';
+import { trigger, state, style, transition, animate } from '@angular/animations';
+import { MatDialog, MatSnackBar } from '@angular/material';
+import { PostDetailComponent } from '../post-detail/post-detail.component';
+import { Observable, Subject, BehaviorSubject, Subscription } from 'rxjs';
+import { BreakpointState, BreakpointObserver, Breakpoints } from '@angular/cdk/layout';
+import { PostState } from './post-state.model';
+import { DownloadPath } from '../common/download-path.model';
+import { ContextMenuService } from '../ctxt-menu.service';
+import { ConfirmDialogComponent } from '../common/confirmation-component/confirmation-dialog';
+import { filter, flatMap } from 'rxjs/operators';
+import { RemoveResponse } from '../common/remove-response.model';
+
+@Component({
+  selector: 'app-post-ctx-menu',
+  template: `
+    <mat-card class="mat-elevation-z4">
+      <mat-action-list>
+        <button
+          (click)="restart()"
+          *ngIf="
+            (postState.status === 'COMPLETE' && postState.progress !== 100) ||
+            postState.status === 'ERROR' ||
+            postState.status === 'STOPPED'
+          "
+          mat-list-item
+        >
+          <mat-icon>play_arrow</mat-icon>
+          <span>Start</span>
+        </button>
+        <button
+          (click)="stop()"
+          *ngIf="postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL' || postState.status === 'PENDING'"
+          mat-list-item
+        >
+          <mat-icon>stop</mat-icon>
+          <span>Stop</span>
+        </button>
+        <button (click)="remove()" mat-list-item>
+          <mat-icon>delete</mat-icon>
+          <span>Remove</span>
+        </button>
+        <button (click)="seeDetails()" mat-list-item>
+          <mat-icon>list</mat-icon>
+          <span>Files</span>
+        </button>
+        <button *ngIf="isViewEnabled | async" (click)="openGallery()" mat-list-item>
+          <mat-icon>photo_library</mat-icon>
+          <span>View Photos</span>
+        </button>
+        <button (click)="open()" *ngIf="electronService.isElectronApp" mat-list-item>
+          <mat-icon>open_in_new</mat-icon>
+          <span>Download Location</span>
+        </button>
+      </mat-action-list>
+    </mat-card>
+  `,
+  styles: [
+    `
+      mat-card {
+        padding: 0 0 8px 0;
+      }
+      mat-icon {
+        margin-right: 5px;
+      }
+    `
+  ],
+  animations: [
+    trigger('simpleFadeAnimation', [
+      state('in', style({ opacity: 1 })),
+
+      transition(':enter', [style({ opacity: 0 }), animate(300)])
+    ])
+  ],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class PostContextMenuComponent {
+  postState: PostState;
+  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
+  fs;
+  contextMenuService: ContextMenuService;
+  constructor(
+    public electronService: ElectronService,
+    private dialog: MatDialog,
+    private breakpointObserver: BreakpointObserver,
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    private _snackBar: MatSnackBar,
+    private ngZone: NgZone,
+    private appService: AppService
+  ) {
+    if (this.electronService.isElectronApp) {
+      this.fs = this.electronService.remote.require('fs');
+    }
+    this.isViewEnabled.next(this.appService.settings.viewPhotos);
+  }
+
+  isViewEnabled: Subject<boolean> = new BehaviorSubject(false);
+  restart() {
+    this.contextMenuService.closePostCtxtMenu();
+    this.httpClient.post(this.serverService.baseUrl + '/post/restart', [this.postState.postId]).subscribe(
+      () => {
+        this._snackBar.open('Download started', null, {
+          duration: 5000
+        });
+      },
+      error => {
+        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+          duration: 5000
+        });
+      }
+    );
+  }
+
+  remove() {
+    this.contextMenuService.closePostCtxtMenu();
+    this.ngZone.run(() => {
+      this.dialog
+        .open(ConfirmDialogComponent, {
+          maxHeight: '100vh',
+          maxWidth: '100vw',
+          height: '200px',
+          width: '60%',
+          data: { header: 'Confirmation', content: 'Are you sure you want to remove this item ?' }
+        })
+        .afterClosed()
+        .pipe(
+          filter(e => e === 'yes'),
+          flatMap(e =>
+            this.httpClient.post<RemoveResponse>(this.serverService.baseUrl + '/post/remove', [this.postState.postId])
+          )
+        )
+        .subscribe(
+          () => {},
+          error => {
+            this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+              duration: 5000
+            });
+          }
+        );
+    });
+  }
+
+  stop() {
+    this.contextMenuService.closePostCtxtMenu();
+    this.httpClient.post(this.serverService.baseUrl + '/post/stop', [this.postState.postId]).subscribe(
+      () => {
+        this._snackBar.open('Download stopped', null, {
+          duration: 5000
+        });
+      },
+      error => {
+        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+          duration: 5000
+        });
+      }
+    );
+  }
+
+  seeDetails() {
+    this.contextMenuService.closePostCtxtMenu();
+    this.ngZone.run(() => {
+      const dialogRef = this.dialog.open(PostDetailComponent, {
+        width: '90%',
+        height: '90%',
+        maxWidth: '100vw',
+        maxHeight: '100vh',
+        data: this.postState
+      });
+
+      const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
+        if (result.matches) {
+          dialogRef.updateSize('100%', '100%');
+        } else {
+          dialogRef.updateSize('90%', '90%');
+        }
+      });
+
+      dialogRef.afterClosed().subscribe(() => {
+        if (smallDialogSubscription != null) {
+          smallDialogSubscription.unsubscribe();
+        }
+      });
+    });
+  }
+
+  openGallery() {
+    this.contextMenuService.closePostCtxtMenu();
+    this.ngZone.run(() => {
+      const dialogRef = this.dialog.open(GalleryComponent, {
+        width: '90%',
+        height: '90%',
+        maxWidth: '100vw',
+        maxHeight: '100vh',
+        data: this.postState
+      });
+
+      const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
+        if (result.matches) {
+          dialogRef.updateSize('100%', '100%');
+        } else {
+          dialogRef.updateSize('90%', '90%');
+        }
+      });
+
+      dialogRef.afterClosed().subscribe(() => {
+        if (smallDialogSubscription != null) {
+          smallDialogSubscription.unsubscribe();
+        }
+      });
+    });
+  }
+
+  open() {
+    this.contextMenuService.closePostCtxtMenu();
+    if (!this.electronService.isElectronApp) {
+      console.error('Cannot open downloader folder, not electron app');
+      return;
+    }
+    // Request the server to give the correct file location
+    this.httpClient.get<DownloadPath>(this.serverService.baseUrl + '/post/path/' + this.postState.postId).subscribe(
+      path => {
+        if (this.fs.existsSync(path.path)) {
+          this.electronService.shell.openItem(path.path);
+        } else {
+          if (this.postState.done <= 0) {
+            this._snackBar.open('Download has not been started yet for this post', null, {
+              duration: 5000
+            });
+          } else {
+            this._snackBar.open(path.path + ' does not exist, you probably removed it', null, {
+              duration: 5000
+            });
+          }
+        }
+      },
+      error => {
+        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+          duration: 5000
+        });
+      }
+    );
+  }
+}

vripper-ui/src/app/posts/post-progress.renderer.component.html

@@ -1,116 +1,10 @@
-<!-- <div class="container" style="height: 100%; background-color: white">
-  <div
-    [ngClass]="{
-      error: postState.status === 'ERROR',
-      downloading: postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL',
-      complete: postState.status === 'COMPLETE',
-      stopped: postState.status === 'STOPPED'
-    }"
-    class="progress-bar-back"
-    style="position: relative; height: 100%; max-height: 48px;"
-  >
-    <div
-      class="progress-foreground"
-      fxLayout="row"
-      fxLayoutAlign="space-between"
-      style="background-color: transparent; width: 100%; height: 100%;"
-    >
-      <div class="progress-bar" color="accent" style="position: absolute; width: 100%; top: 37px; padding: 0 50px">
-        <mat-progress-bar [value]="postState.progress" class="example-margin" color="primary" mode="determinate">
-        </mat-progress-bar>
-      </div>
-      <span fxLayout="row">
-        <span (click)="toggleExpand()" class="chevron">
-          <button *ngIf="expanded; else notExpanded" mat-icon-button>
-            <mat-icon>expand_more</mat-icon>
-          </button>
-          <ng-template #notExpanded>
-            <button mat-icon-button>
-              <mat-icon>chevron_right</mat-icon>
-            </button>
-          </ng-template>
-        </span>
-        <span>{{ postState.title }}</span>
-        <span class="filler" fxFlex="grow"></span>
-      </span>
-      <span>
-        <span class="progress-percentage">{{ postState.done + '/' + postState.total }}</span>
-        <button [matMenuTriggerFor]="menu" mat-icon-button>
-          <mat-icon>more_vert</mat-icon>
-        </button>
-        <mat-menu #menu="matMenu">
-          <button
-            (click)="restart()"
-            *ngIf="
-              (postState.status === 'COMPLETE' && postState.progress !== 100) ||
-              postState.status === 'ERROR' ||
-              postState.status === 'STOPPED'
-            "
-            mat-menu-item
-          >
-            <mat-icon>play_arrow</mat-icon>
-            <span>Start</span>
-          </button>
-          <button
-            (click)="stop()"
-            *ngIf="
-              postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL' || postState.status === 'PENDING'
-            "
-            mat-menu-item
-          >
-            <ng-container>
-              <mat-icon>stop</mat-icon>
-              <span>Stop</span>
-            </ng-container>
-          </button>
-          <button (click)="seeDetails()" mat-menu-item>
-            <mat-icon>list</mat-icon>
-            <span>Files</span>
-          </button>
-          <button (click)="remove()" mat-menu-item>
-            <mat-icon>delete</mat-icon>
-            <span>Remove</span>
-          </button>
-          <button (click)="open()" *ngIf="electronService.isElectronApp" mat-menu-item>
-            <mat-icon>open_in_new</mat-icon>
-            <span>Download Location</span>
-          </button>
-        </mat-menu>
-      </span>
-    </div>
-  </div>
-  <section *ngIf="expanded">
-    <div class="details table">
-      <div class="row" style="display: table-row">
-        <h4 class="cell attribute">Status:</h4>
-        <p class="cell value">
-          {{ postState.status | titlecase }}
-        </p>
-      </div>
-      <div class="row" style="display: table-row">
-        <h4 class="cell attribute">Post URL:</h4>
-        <p class="cell value">
-          <a (click)="goTo()" [appPreview]="postState.previews" href="javascript:void(0)"
-            >https://vipergirls/threads/?p={{ postState.postId }}</a
-          >
-        </p>
-      </div>
-      <div style="display: table-row">
-        <h4 class="cell attribute">Host:</h4>
-        <p class="cell value">
-          {{ postState.hosts }}
-        </p>
-      </div>
-    </div>
-  </section>
-</div> -->
-
-<div class="container" style="height: 100%; background-color: white">
+<div (contextmenu)="context($event)" class="container" style="height: 100%; background-color: white">
   <ng-container *ngIf="postState$ | async as postState">
     <div
       [ngClass]="{
         error: postState.status === 'ERROR',
-        downloading: postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL',
+        downloading: postState.status === 'DOWNLOADING',
+        partial: postState.status === 'PARTIAL',
         complete: postState.status === 'COMPLETE',
         stopped: postState.status === 'STOPPED'
       }"
@@ -124,31 +18,21 @@
         fxLayoutGap="5px"
         style="background-color: transparent; width: 100%; height: 100%;"
       >
-        <div class="checkbox" style="display: inline-block; width: 5px; height: 100%;"></div>
-        <span fxFlex="grow" fxLayout="row" fxLayoutAlign="space-between center">
-          <span
-          ><a (click)="goTo()" [appPreview]="postState.previews" href="javascript:void(0)">{{
+        <div class="checkbox" fxFlex="nogrow" style="width: 5px; height: 100%;"></div>
+        <span class="row-text" fxFlex="grow" fxLayout="row" fxLayoutAlign="space-between center" fxLayoutGap="20px">
+          <span [title]="postState.title">
+            <a (click)="goTo()" [appPreview]="postState.previews" href="javascript:void(0)">{{
               postState.title
             }}</a></span
           >
-          <span>{{postState.done + '/' + postState.total}} done from {{postState.hosts}}</span>
+          <span [title]="postState.done + '/' + postState.total + ' from ' + postState.hosts"
+          >{{ postState.done + '/' + postState.total }} from {{ postState.hosts }}</span
+          >
         </span>
-        <button [matMenuTriggerFor]="menu" mat-icon-button>
-          <mat-icon>more_vert</mat-icon>
-        </button>
-        <mat-menu #menu="matMenu">
-          <button (click)="seeDetails()" mat-menu-item>
-            <mat-icon>list</mat-icon>
-            <span>Files</span>
-          </button>
-          <button (click)="open()" *ngIf="electronService.isElectronApp" mat-menu-item>
-            <mat-icon>open_in_new</mat-icon>
-            <span>Download Location</span>
-          </button>
-        </mat-menu>
       </div>
       <div class="progress-bar" color="accent" style="position: absolute; width: 100%; top: 37px; padding: 0 10px">
-        <mat-progress-bar [value]="postState.progress" color="primary" mode="determinate"></mat-progress-bar>
+        <mat-progress-bar *ngIf="(loaded | async)" [value]="postState.progress" color="primary"
+                          mode="determinate"></mat-progress-bar>
       </div>
     </div>
   </ng-container>

vripper-ui/src/app/posts/post-progress.renderer.component.scss

@@ -1,3 +1,15 @@
 .checkbox {
     background-color: transparent;
 }
+
+.row-text {
+    white-space: nowrap; 
+    overflow: hidden;
+    padding-right: 15px;
+}
+
+.row-text > span {
+    white-space: nowrap; 
+    overflow: hidden;
+    text-overflow: ellipsis;
+}

vripper-ui/src/app/posts/post-progress.renderer.component.scss-theme.scss

@@ -4,6 +4,7 @@
   $red: mat-palette($mat-red);
   $grey: mat-palette($mat-grey);
   $green: mat-palette($mat-green);
+  $orange: mat-palette($mat-orange);
 
   app-progress-cell .progress-bar .mat-progress-bar-fill::after {
     background-color: mat-color($grey, 400);
@@ -14,7 +15,7 @@
   }
 
   app-progress-cell .error {
-    background-color: mat-color($red, 50);
+    background-color: mat-color($red, 100);
     & .progress-bar .mat-progress-bar-fill::after {
       background-color: mat-color($red, 300);
     }
@@ -23,6 +24,16 @@
     }
   }
 
+  app-progress-cell .partial {
+    background-color: mat-color($orange, 100);
+    & .progress-bar .mat-progress-bar-fill::after {
+      background-color: mat-color($orange, 300);
+    }
+    & .progress-bar .mat-progress-bar-buffer {
+      background-color: mat-color($orange, 100);
+    }
+  }
+
   app-progress-cell .downloading {
     background-color: mat-color($green, 50);
     & .progress-bar .mat-progress-bar-fill::after {
@@ -44,6 +55,6 @@
   }
 
   app-progress-cell .stopped {
-    background-color: mat-color($grey, 100);
+    background-color: mat-color($grey, 300);
   }
 }

vripper-ui/src/app/posts/post-progress.renderer.component.ts

@@ -7,19 +7,15 @@ import {
   NgZone,
   ChangeDetectionStrategy,
   AfterViewInit,
-  EventEmitter
+  EventEmitter,
 } from '@angular/core';
 import { AgRendererComponent } from 'ag-grid-angular';
-import { Subscription, Observable } from 'rxjs';
+import { Subscription, BehaviorSubject, Subject } from 'rxjs';
 import { WsHandler } from '../ws-handler';
-import { ICellRendererParams } from 'ag-grid-community';
+import { ICellRendererParams, RowNode, GridApi } from 'ag-grid-community';
 import { ElectronService } from 'ngx-electron';
-import { BreakpointState, Breakpoints, BreakpointObserver } from '@angular/cdk/layout';
-import { MatDialog, MatSnackBar } from '@angular/material';
-import { PostDetailComponent } from '../post-detail/post-detail.component';
-import { HttpClient } from '@angular/common/http';
-import { ServerService } from '../server-service';
-import { DownloadPath } from '../common/download-path.model';
+import { MatDialog } from '@angular/material';
+import { ContextMenuService } from '../ctxt-menu.service';
 
 @Component({
   selector: 'app-progress-cell',
@@ -32,16 +28,9 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
     private wsConnectionService: WsConnectionService,
     private zone: NgZone,
     public electronService: ElectronService,
-    private breakpointObserver: BreakpointObserver,
     public dialog: MatDialog,
-    private httpClient: HttpClient,
-    private serverService: ServerService,
-    private _snackBar: MatSnackBar
-  ) {
+    private contextMenuService: ContextMenuService) {
     this.websocketHandlerPromise = this.wsConnectionService.getConnection();
-    if (this.electronService.isElectronApp) {
-      this.fs = this.electronService.remote.require('fs');
-    }
   }
 
   websocketHandlerPromise: Promise<WsHandler>;
@@ -49,8 +38,10 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
   private postState: PostState;
   updatesSubscription: Subscription;
   expanded = false;
-  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
-  fs;
+  loaded: Subject<boolean> = new BehaviorSubject(false);
+  loading;
+  node: RowNode;
+  gridApi: GridApi;
 
   trunc(value: number): number {
     return Math.trunc(value);
@@ -73,15 +64,19 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
 
   ngAfterViewInit(): void {
     this.postState$.emit(this.postState);
+    this.loading = setTimeout(() => this.loaded.next(true), 100);
   }
 
   ngOnDestroy(): void {
     if (this.updatesSubscription != null) {
       this.updatesSubscription.unsubscribe();
     }
+    clearTimeout(this.loading);
   }
 
   agInit(params: ICellRendererParams): void {
+    this.node = params.node;
+    this.gridApi = params.api;
     this.postState = params.data;
   }
 
@@ -99,55 +94,9 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
     }
   }
 
-  seeDetails() {
-    const dialogRef = this.dialog.open(PostDetailComponent, {
-      width: '90%',
-      height: '90%',
-      maxWidth: '100vw',
-      maxHeight: '100vh',
-      data: this.postState
-    });
-
-    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
-      if (result.matches) {
-        dialogRef.updateSize('100%', '100%');
-      } else {
-        dialogRef.updateSize('90%', '90%');
-      }
-    });
-
-    dialogRef.afterClosed().subscribe(() => {
-      if (smallDialogSubscription != null) {
-        smallDialogSubscription.unsubscribe();
-      }
-    });
-  }
-
-  open() {
-    if (!this.electronService.isElectronApp) {
-      console.error('Cannot open downloader folder, not electron app');
-      return;
-    }
-    // Request the server to give the correct file location
-    this.httpClient.get<DownloadPath>(this.serverService.baseUrl + '/post/path/' + this.postState.postId).subscribe(
-      path => {
-        if (this.fs.existsSync(path.path)) {
-          this.electronService.shell.openItem(path.path);
-        } else {
-          if (this.postState.done <= 0) {
-            this._snackBar.open('Download has not been started yet for this post', null, {
-              duration: 5000
-            });
-          } else {
-            this._snackBar.open(path.path + ' does not exist, you probably removed it', null, {
-              duration: 5000
-            });
-          }
-        }
-      },
-      error => {
-        console.error(error);
-      }
-    );
+  context(event: MouseEvent) {
+    this.gridApi.getSelectedNodes().forEach(e => e.setSelected(false));
+    this.node.setSelected(true);
+    this.contextMenuService.openPostCtxtMenu(event, this.postState);
   }
 }

vripper-ui/src/app/posts/posts.component.ts

@@ -6,6 +6,7 @@ import { WsConnectionService } from '../ws-connection.service';
 import { GridOptions } from 'ag-grid-community';
 import { PostProgressRendererComponent } from './post-progress.renderer.component';
 import { Subject } from 'rxjs';
+import { ContextMenuService } from '../ctxt-menu.service';
 
 @Component({
   selector: 'app-posts',
@@ -17,7 +18,8 @@ export class PostsComponent implements OnInit, OnDestroy, AfterViewInit {
     private wsConnection: WsConnectionService,
     private zone: NgZone,
     private selectionService: SelectionService,
-    private postsDataService: PostsDataService
+    private postsDataService: PostsDataService,
+    private contextMenuService: ContextMenuService
   ) {
     this.gridOptions = <GridOptions>{
       columnDefs: [
@@ -28,10 +30,12 @@ export class PostsComponent implements OnInit, OnDestroy, AfterViewInit {
           cellRenderer: 'progressCellRenderer',
           cellClass: 'no-padding',
           sort: 'asc',
+          suppressMovable: true,
           headerCheckboxSelection: true,
           headerCheckboxSelectionFilteredOnly: true
         }
       ],
+      rowBuffer: 200,
       rowHeight: 48,
       animateRows: true,
       rowSelection: 'multiple',
@@ -50,7 +54,8 @@ export class PostsComponent implements OnInit, OnDestroy, AfterViewInit {
       },
       onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
       onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit(),
-      onSelectionChanged: () => this.selectionService.onSelectionChanged(this.gridOptions.api.getSelectedNodes())
+      onSelectionChanged: () => this.selectionService.onSelectionChanged(this.gridOptions.api.getSelectedNodes()),
+      onBodyScroll: () => this.contextMenuService.closePostCtxtMenu()
     };
   }
 

vripper-ui/src/app/scan/scan.component.html

@@ -3,32 +3,24 @@
     <h2 class="no-wrap" mat-dialog-title>Scan</h2>
   </div>
   <mat-dialog-content fxFlex="grow" fxLayout="column">
-    <div *ngIf="!(hideScan | async)">
-      <form
-        #f="ngForm"
-        (ngSubmit)="submit(f)"
-        autocomplete="off"
-        fxLayout="row"
-        fxLayoutAlign="center center"
-        fxLayoutGap="20px"
-      >
-        <mat-form-field fxFlex="grow">
-          <input [(ngModel)]="input" matInput name="url" placeholder="Put a vipergirls.to link" required/>
-        </mat-form-field>
-        <div>
-          <button [disabled]="f.invalid" color="primary" mat-raised-button type="submit">Scan</button>
-        </div>
-      </form>
-    </div>
-    <ng-container *ngIf="threadId | async as thId">
-      <app-multi-post (done)="done($event)" *ngIf="thId != null" [threadId]="thId"
-                      style="height: 100%"></app-multi-post>
-    </ng-container>
+    <form #f="ngForm" autocomplete="off" style="height: 100%;">
+      <mat-form-field fxFlex="grow">
+        <textarea
+          [(ngModel)]="input"
+          mat-autosize="true"
+          matAutosizeMinRows="5"
+          matInput
+          name="url"
+          placeholder="Put each link in a new line"
+          required
+        ></textarea>
+      </mat-form-field>
+    </form>
+    <mat-progress-bar *ngIf="loading | async" mode="indeterminate"></mat-progress-bar>
   </mat-dialog-content>
-  <mat-dialog-actions align="end">
-    <button (click)="close()" mat-raised-button>Close</button>
-    <button (click)="addPosts()" *ngIf="threadId != null" color="primary" mat-raised-button type="submit">
-      Download
+  <mat-dialog-actions align="end" fxFlex="nogrow">
+    <button mat-dialog-close mat-raised-button>Close</button>
+    <button (click)="submit()" [disabled]="f.invalid || (loading | async)" color="primary" mat-raised-button>Scan
     </button>
   </mat-dialog-actions>
 </div>

vripper-ui/src/app/scan/scan.component.ts

@@ -2,10 +2,8 @@ import {
   Component,
   OnInit,
   NgZone,
-  ViewChild,
   Inject,
   ChangeDetectionStrategy,
-  EventEmitter,
   AfterViewInit
 } from '@angular/core';
 import { NgForm } from '@angular/forms';
@@ -13,7 +11,7 @@ import { MatSnackBar, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material';
 import { HttpClient } from '@angular/common/http';
 import { finalize } from 'rxjs/operators';
 import { ServerService } from '../server-service';
-import { MultiPostComponent } from '../multi-post/multi-post.component';
+import { Subject, BehaviorSubject } from 'rxjs';
 
 @Component({
   selector: 'app-scan',
@@ -31,18 +29,12 @@ export class ScanComponent implements OnInit, AfterViewInit {
     @Inject(MAT_DIALOG_DATA) public data: DialogData
   ) {}
 
-  @ViewChild(MultiPostComponent)
-  multipost: MultiPostComponent;
-
   input: string;
-  threadId: EventEmitter<string> = new EventEmitter();
-  hideScan: EventEmitter<boolean> = new EventEmitter();
+  loading: Subject<boolean> = new BehaviorSubject(false);
 
-  submit(form: NgForm) {
+  submit() {
     this.ngZone.run(() => {
-      this.hideScan.emit(true);
-      this.threadId.emit(null);
-      this.processUrl(this.input, form);
+      this.processUrl(this.input);
     });
   }
 
@@ -54,62 +46,30 @@ export class ScanComponent implements OnInit, AfterViewInit {
     });
   }
 
-  processUrl(url: string, form?: NgForm) {
+  processUrl(url: string) {
+    this.loading.next(true);
     this.httpClient
       .post<{ threadId: string; postId: string }>(this.serverService.baseUrl + '/post', { url: url })
       .pipe(
         finalize(() => {
-          this.ngZone.run(() => {
-            if (form != null) {
-              form.resetForm();
-              this.input = null;
-            }
-          });
+          this.close();
+          this.loading.next(false);
         })
       )
-      .subscribe(response => {
-        this.ngZone.run(() => {
-          if (response.postId != null) {
-            this.httpClient
-              .post(this.serverService.baseUrl + '/post/add', [response])
-              .pipe(finalize(() => this.dialogRef.close()))
-              .subscribe(
-                () => {
-                  this._snackBar.open('Adding posts to queue', null, {
-                    duration: 5000
-                  });
-                },
-                error => {
-                  this._snackBar.open(error.error, null, {
-                    duration: 5000
-                  });
-                }
-              );
-            return;
-          }
-          this.threadId.emit(response.threadId);
+      .subscribe(response => {},
+      error => {
+        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+          duration: 5000
         });
       });
   }
 
-  addPosts() {
-    this.ngZone.run(() => {
-      if (this.multipost != null) {
-        this.multipost.submit();
-      }
-      this.dialogRef.close();
-    });
-  }
-
   ngOnInit() {}
 
   ngAfterViewInit(): void {
     this.ngZone.run(() => {
       if (this.data.url != null) {
-        this.hideScan.emit(true);
         this.processUrl(this.data.url);
-      } else {
-        this.hideScan.emit(false);
       }
     });
   }

vripper-ui/src/app/settings/settings.component.html

@@ -25,25 +25,43 @@
               </button>
             </div>
 
-            <mat-form-field>
-              <input
-                formControlName="maxThreads"
-                type="number"
-                matInput
-                placeholder="Max concurrent downloads"
-                name="maxThreads"
-                min="1"
-                max="4"
-                required
-              />
-            </mat-form-field>
+            <div fxLayout="row" fxLayoutAlign="center center" fxLayoutGap="10px">
+              <mat-form-field fxFlex="grow">
+                <input
+                  formControlName="maxThreads"
+                  matInput
+                  max="4"
+                  min="1"
+                  name="maxThreads"
+                  placeholder="Max concurrent downloads per host"
+                  required
+                  type="number"
+                />
+              </mat-form-field>
+
+              <mat-form-field fxFlex="grow">
+                <input
+                  formControlName="maxTotalThreads"
+                  matInput
+                  min="0"
+                  name="maxTotalThreads"
+                  placeholder="Max global concurrent downloads"
+                  required
+                  type="number"
+                />
+              </mat-form-field>
+            </div>
 
             <mat-checkbox color="primary" formControlName="subLocation" name="subLocation"
             >Save posts in sub folders
             </mat-checkbox>
 
-            <mat-checkbox *ngIf="generalSettingsForm.get('subLocation').value" color="primary"
-                          formControlName="threadSubLocation" name="threadSubLocation" style="margin-left: 15px"
+            <mat-checkbox
+              *ngIf="generalSettingsForm.get('subLocation').value"
+              color="primary"
+              formControlName="threadSubLocation"
+              name="threadSubLocation"
+              style="margin-left: 15px"
             >Create a subfolder per thread
             </mat-checkbox>
 
@@ -52,13 +70,24 @@
             </mat-checkbox>
 
             <mat-checkbox color="primary" formControlName="autoStart" name="autoStart"
-              >Auto start downloads</mat-checkbox
-            >
+            >Auto start downloads
+            </mat-checkbox>
 
             <mat-checkbox color="primary" formControlName="clearCompleted" name="clearCompleted"
             >Clear Finished
             </mat-checkbox>
 
+            <mat-checkbox color="primary" formControlName="viewPhotos" name="viewPhotos">
+              Enable photo gallery
+            </mat-checkbox>
+            <div style="margin-left: 15px">
+              Cache size: {{(cacheSize | async).size}}
+              <button (click)="clearCache()" [disabled]="(cacheClearLoading | async)" color="primary"
+                      mat-stroked-button>
+                <mat-icon>clear</mat-icon>
+                <span>Clear</span></button>
+            </div>
+
             <mat-slide-toggle color="primary" formControlName="vLogin" name="vLogin">
               ViperGirls Authentication
             </mat-slide-toggle>
@@ -66,7 +95,7 @@
             <section *ngIf="generalSettingsForm.get('vLogin').value">
               <div>
                 <mat-form-field>
-                  <input formControlName="vUsername" matInput placeholder="ViperGirls Username" name="vUsername" />
+                  <input formControlName="vUsername" matInput name="vUsername" placeholder="ViperGirls Username"/>
                 </mat-form-field>
 
                 <mat-form-field>
@@ -84,10 +113,17 @@
             </section>
           </form>
         </mat-tab>
-        <mat-tab label="Desktop Integration" *ngIf="electronService.isElectronApp">
+        <mat-tab label="Desktop Integration">
           <form [formGroup]="desktopSettingsForm" autocomplete="off">
-            <mat-checkbox color="primary" formControlName="desktopClipboard" name="desktopClipboard"
-              >Monitor Clipboard</mat-checkbox
+            <mat-checkbox *ngIf="electronService.isElectronApp" color="primary" formControlName="desktopClipboard"
+                          name="desktopClipboard"
+            >Monitor Clipboard
+            </mat-checkbox
+            >
+
+            <mat-checkbox color="primary" formControlName="notification" name="notification"
+            >Enable system notifications
+            </mat-checkbox
             >
           </form>
         </mat-tab>
@@ -104,7 +140,8 @@
       [disabled]="
         (generalSettingsForm.pristine && desktopSettingsForm.pristine) ||
         generalSettingsForm.invalid ||
-        desktopSettingsForm.invalid
+        desktopSettingsForm.invalid ||
+        loading
       "
     >
       Apply