v2.10.3

Enhance download queue logic
Clear cache button
Fix the partial status

CHANGELOG.md

@@ -1,5 +1,117 @@
 # Changelog
 
+## [2.10.3] - 2020-01-16
+### Changed
+- Some bug fixes
+- Enhance download queue logic
+- Clear cache button
+- Fix the partial status
+
+## [2.10.2] - 2019-12-29
+### Changed
+- Upgrade electron to v7
+- Bug fix
+
+## [2.10.1] - 2019-12-27
+### Changed
+- Bug fix
+
+## [2.10.0] - 2019-12-27
+### Changed
+- Fix spring dependency bug
+### Added
+- Add support for imgspice
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Fix build issue
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Decrease timeout for fast fail
+### Added
+- Allow many hosts to run at the same time
+- Add title to gallery
+- Add global concurrent downloads
+- Add counter badge in link collector
+
+## [2.8.1] - 2019-12-24
+### Changed
+- Fix issue with path creation
+
+## [2.8.0] - 2019-12-24
+### Added
+- Photo gallery feature
+
+## [2.7.1] - 2019-12-21
+### Added
+- Add version to title bar
+
+## [2.7.0] - 2019-12-21
+### Changed
+- Add missing icon
+### Added
+- Add support for pixxxels
+
+## [2.6.0] - 2019-12-21
+### Changed
+- Disable download button when nothing selected in Link collector grab menu
+- Better quality app icon
+- Implement context menu on download tab
+- Add stop remove and start button to context menu
+- Link will not be removed automatically from link collector tab
+- Some UI fixes
+
+## [2.5.1] - 2019-12-09
+### Changed
+- Disable download button when nothing selected in Link collector grab menu
+### Added
+- Add hosts column in Link collector grab menu
+
+## [2.5.0] - 2019-12-08
+### Changed
+- Change loading screen
+- UI Fixes
+- Fix bug with previews
+### Added
+- Add link collector feature
+- Add support for multiple links at a time
+
+## [2.4.1] - 2019-11-26
+### Changed
+- Increase attempts for establishing ws connection between front and back end
+- Change default download folder to Home folder (installer), App folder (portable)
+
+## [2.4.0] - 2019-11-23
+### Changed
+- Pimpandhost Host support
+
+## [2.3.2] - 2019-11-23
+### Changed
+- Fix a bug with the scan component
+
+## [2.3.1] - 2019-11-22
+### Changed
+- Fix connexion reset issue
+- Fix Post Status when autostart option is selected
+- Add stop all button
+
+## [2.3.0] - 2019-11-21
+### Changed
+- UI enhancements and overhaul
+- Stability fixes with HTTP connections
+- MacOS official support
+- Use local sockjs and material icons font
+
+## [2.2.5] - 2019-11-09
+### Changed
+- Exclude thumbs download for imagezilla and pixhost
+- Change retry on fail logic (may help avoid having download errors)
+
+## [2.2.4] - 2019-10-05
+### Added
+- AppImage target for linux
+
 ## [2.2.3] - 2019-10-01
 ### Changed
 - Strip libjvm.so for debian

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.2.3</version>
+    <version>2.10.3</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/main.js

@@ -4,7 +4,6 @@ const url = require("url");
 const getPort = require("get-port");
 const { spawn } = require("child_process");
 const { ipcMain } = require("electron");
-const commandExists = require("command-exists").sync;
 const { dialog } = require("electron");
 const axios = require('axios');
 const appDir = process.env.APPDIR;
@@ -23,14 +22,6 @@ process.on("uncaughtException", err => {
   process.exit(1);
 });
 
-if (!commandExists("java")) {
-  dialog.showErrorBox(
-    "Java command is missing",
-    "Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
-  );
-  process.exit(1);
-}
-
 function createWindow() {
   let icon;
   if(process.platform === "win32") {
@@ -73,14 +64,31 @@ getPort().then(port => {
   ipcMain.on("get-port", event => {
     event.reply("port", port);
   });
-  vripperServer = spawn(appDir !== undefined ? path.join(appDir, "java-runtime/bin/java") :
-  path.join(app.getPath('exe'), "../java-runtime/bin/java"), [
+  let javaBinPath;
+  if(appDir !== undefined) {
+    javaBinPath = path.join(appDir, "java-runtime/bin/java");
+  } else {
+    if(process.platform === 'darwin') {
+      javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+    } else {
+      javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+    }
+  }
+  let jarPath;
+  if(appDir !== undefined) {
+    jarPath = path.join(appDir, "bin/vripper-server.jar");
+  } else {
+    if(process.platform === 'darwin') {
+      jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+    } else {
+      jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+    }
+  }
+  vripperServer = spawn(javaBinPath, [
     "-Xms256m",
-    "-Xmx1024m",
     "-Dvripper.server.port=" + port,
     "-jar",
-    appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
-    path.join(app.getPath('exe'), "../bin/vripper-server.jar")
+    jarPath
   ], {
     stdio: 'ignore'
   });
@@ -127,6 +135,7 @@ if (!gotTheLock) {
       })
       .catch((error) => {
         // Terminate immediately
+        console.log(error);
         vripperServer.kill('SIGKILL');
         terminated = true;
         app.quit();

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.2.3",
+  "version": "2.10.3",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -8,7 +8,7 @@
   "license": "ISC",
   "build": {
     "appId": "tn.mnlr.vripper",
-    "productName": "vripper",
+    "productName": "VRipper",
     "files": [
       "**/*",
       "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -38,7 +38,7 @@
       }
     ],
     "win": {
-      "icon": "icon.png",
+      "icon": "icon.ico",
       "target": [
         "nsis"
       ]
@@ -54,18 +54,15 @@
       "packageCategory": "Utility",
       "icon": "icons",
       "target": [
-        "deb"
+        "AppImage"
       ]
     },
-    "deb": {
-      "depends": [
-        "gconf2",
-        "gconf-service",
-        "libnotify4",
-        "libappindicator1",
-        "libxtst6",
-        "libnss3"
-      ]
+    "mac": {
+      "category": "public.app-category.utilities",
+      "target": [
+        "dmg"
+      ],
+      "icon": "icon.icns"
     }
   },
   "scripts": {
@@ -73,15 +70,13 @@
     "dist": "node pre-build.js && electron-builder"
   },
   "devDependencies": {
-    "electron": "^6.0.9",
+    "electron": "^7.1.7",
     "electron-builder": "^21.2.0"
   },
   "dependencies": {
     "axios": "^0.19.0",
     "cheerio": "^1.0.0-rc.3",
-    "command-exists": "^1.2.8",
     "copy-dir": "^1.1.0",
-    "custom-electron-titlebar": "^3.0.10",
     "electron-context-menu": "^0.13.0",
     "get-port": "^5.0.0",
     "rimraf": "^2.6.3"

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.2.3</version>
+    <version>2.10.3</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>2.2.3</version>
+        <version>2.10.3</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
@@ -67,6 +67,16 @@
             <artifactId>failsafe</artifactId>
             <version>2.0.1</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>28.1-jre</version>
+        </dependency>
+        <dependency>
+            <groupId>org.imgscalr</groupId>
+            <artifactId>imgscalr-lib</artifactId>
+            <version>4.2</version>
+        </dependency>
         <dependency>
             <groupId>tn.mnlr</groupId>
             <artifactId>vripper-ui</artifactId>

vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java

@@ -0,0 +1,24 @@
+package tn.mnlr.vripper;
+
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.event.ContextRefreshedEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+import tn.mnlr.vripper.services.PersistenceService;
+
+@Component
+public class EventListenerBean {
+
+    @Autowired
+    private PersistenceService persistenceService;
+
+    @Getter
+    private static boolean init = false;
+
+    @EventListener
+    public void onApplicationEvent(ContextRefreshedEvent event) {
+        init = true;
+        persistenceService.restore();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/SpringContext.java

@@ -17,24 +17,19 @@ public class SpringContext implements ApplicationContextAware {
 
     private static ConfigurableApplicationContext context;
 
-    /**
-     * Returns the Spring managed bean instance of the given class type (if it exists).
-     * Returns null otherwise.
-     *
-     * @param beanClass
-     * @return
-     */
-    public static <T extends Object> T getBean(Class<T> beanClass) {
+    public static <T> T getBean(Class<T> beanClass) {
         return context.getBean(beanClass);
     }
 
-    public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
+    public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
         return context.getBeansOfType(beanClass);
     }
 
     public static void close() {
         logger.info("Application terminating...");
-        context.close();
+        if (context != null) {
+            context.close();
+        }
 
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java

@@ -1,21 +1,10 @@
 package tn.mnlr.vripper;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.stereotype.Component;
-import tn.mnlr.vripper.exception.VripperException;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.PersistenceService;
-import tn.mnlr.vripper.services.VipergirlsAuthService;
 
-import javax.annotation.PostConstruct;
-import java.io.File;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -23,59 +12,15 @@ import java.util.concurrent.Executors;
 public class VripperApplication {
 
     private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
-
-//    public static final String dataPath = System.getProperty("user.home", ".") + File.separator + ".vripper" + File.separator + "data.json";
-
     public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
     public static void main(String[] args) {
-
         try {
+            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
             SpringApplication.run(VripperApplication.class, args);
         } catch (Exception e) {
             logger.error("Failed to run the application", e);
         }
     }
-
-    @Component
-    public class AppCommandRunner implements CommandLineRunner {
-
-        @Autowired
-        private VipergirlsAuthService authService;
-
-        @Autowired
-        private PersistenceService persistenceService;
-
-        @Autowired
-        private AppSettingsService appSettingsService;
-
-        @Value("${base.dir}")
-        private String baseDir;
-
-        @Getter
-        private String dataPath;
-
-        @PostConstruct
-        public void init() {
-            dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
-        }
-
-        @Override
-        public void run(String... args) {
-
-            persistenceService.restore();
-            appSettingsService.restore();
-            registerShutdownHook();
-
-            try {
-                authService.authenticate();
-            } catch (VripperException e) {
-                logger.error("Cannot authenticate user with ViperGirls", e);
-            }
-        }
-
-        private void registerShutdownHook() {
-            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
-        }
-    }
 }
+

vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java

@@ -48,7 +48,7 @@ public class Image {
         this.postName = postName;
         this.host = host;
         this.index = index;
-        status = Status.PENDING;
+        status = Status.STOPPED;
         imageStateProcessor = BehaviorProcessor.create();
     }
 
@@ -80,7 +80,7 @@ public class Image {
                 .subscribe();
 
         current.set(0);
-        status = Status.PENDING;
+        status = Status.STOPPED;
         imageStateProcessor.onNext(this);
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java

@@ -57,20 +57,9 @@ public class Post {
         this.forum = forum;
         this.threadId = threadId;
         this.threadTitle = threadTitle;
-        if (this.images.contains(null)) {
-            System.out.println("Oops");
-        }
-        this.images.stream().map(e -> {
-            if (e == null) {
-                System.out.println("Am the cause");
-            }
-            return e.getHost();
-        }).collect(Collectors.toSet());
-        this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
-        this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
+        this.hosts = this.images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet());
         total = images.size();
-        status = Status.PENDING;
-
+        status = Status.STOPPED;
     }
 
     public void setAppStateService(AppStateService appStateService) {

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/persistance/ImagePersistanceMixin.java

@@ -5,10 +5,14 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.BehaviorProcessor;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class ImagePersistanceMixin {
 
     @JsonSerialize(converter = HostToString.class)

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/persistance/PostPersistanceMixin.java

@@ -1,8 +1,12 @@
 package tn.mnlr.vripper.entities.mixin.persistance;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class PostPersistanceMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/ui/ImageUIMixin.java

@@ -3,10 +3,14 @@ package tn.mnlr.vripper.entities.mixin.ui;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.BehaviorProcessor;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class ImageUIMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/ui/PostUIMixin.java

@@ -1,11 +1,15 @@
 package tn.mnlr.vripper.entities.mixin.ui;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.List;
 
+@Getter
+@Setter
 public abstract class PostUIMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/exception/DownloadException.java

@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
         super(message);
     }
 
-    public DownloadException(Exception e) {
+    public DownloadException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class HostException extends Exception {
-    public HostException(Exception e) {
+    public HostException(Throwable e) {
         super(e);
     }
 
@@ -9,7 +9,7 @@ public class HostException extends Exception {
         super(message);
     }
 
-    public HostException(String message, Exception e) {
+    public HostException(String message, Throwable e) {
         super(message,e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HtmlProcessorException.java

@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
 
 public class HtmlProcessorException extends Exception {
 
-    public HtmlProcessorException(Exception e) {
+    public HtmlProcessorException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/PostParseException.java

@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
         super(message);
     }
 
-    public PostParseException(String message, Exception e) {
+    public PostParseException(String message, Throwable e) {
         super(message, e);
     }
 
-    public PostParseException(Exception e) {
+    public PostParseException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/VripperException.java

@@ -5,7 +5,7 @@ public class VripperException extends Exception {
         super(message);
     }
 
-    public VripperException(Exception e) {
+    public VripperException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/XpathException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class XpathException extends Exception {
-    public XpathException(Exception e) {
+    public XpathException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java

@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -28,7 +29,7 @@ public class AcidimgHost extends Host {
 
     private static final String host = "acidimg.cc";
     private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
-    public static final String IMG_XPATH = "//img[@class='centred']";
+    private static final String IMG_XPATH = "//img[@class='centred']";
 
     @Autowired
     private ConnectionManager cm;
@@ -39,20 +40,25 @@ public class AcidimgHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node contDiv;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
-            logger.info(String.format("Click button found for %s", url));
+            logger.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpPost httpPost = cm.buildHttpPost(url);
             httpPost.addHeader("Referer", url);
@@ -64,9 +70,9 @@ public class AcidimgHost extends Host {
                 throw new HostException(e);
             }
 
-            logger.info(String.format("Requesting %s", httpPost));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
-                logger.info(String.format("Cleaning response for %s", httpPost));
+            logger.debug(String.format("Requesting %s", httpPost));
+            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
+                logger.debug(String.format("Cleaning response for %s", httpPost));
                 doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
                 EntityUtils.consumeQuietly(response.getEntity());
             } catch (Exception e) {
@@ -76,7 +82,7 @@ public class AcidimgHost extends Host {
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
@@ -87,7 +93,7 @@ public class AcidimgHost extends Host {
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -5,6 +5,8 @@ import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,9 +27,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.Iterator;
-import java.util.Random;
+import java.util.Objects;
 
 @Service
 abstract public class Host {
@@ -59,59 +64,67 @@ abstract public class Host {
 
     abstract public String getHost();
 
+    abstract public String getLookup();
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(pathService);
+    }
+
     public boolean isSupported(String url) {
-        return url.contains(getHost());
+        return url.contains(getLookup());
     }
 
     public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
 
+        HttpClientContext context = HttpClientContext.create();
+        context.setCookieStore(new BasicCookieStore());
         try {
 
             appStateService.postDownloadingUpdate(image.getPostId());
 
             imageFileData.setPageUrl(image.getUrl());
 
-            /**
+            /*
              * HOST SPECIFIC
              */
-            logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
-            setNameAndUrl(image.getUrl(), imageFileData);
-            logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
-            logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
+            logger.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
+            setNameAndUrl(image.getUrl(), imageFileData, context);
+            logger.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
+            logger.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
 
-            logger.info(String.format("Building image request for %s", image.getUrl()));
+            logger.debug(String.format("Building image request for %s", image.getUrl()));
             setImageRequest(imageFileData);
-            /**
+            /*
              * END HOST SPECIFIC
              */
 
             String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
-            logger.info(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
+            logger.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
             imageFileData.setImageName(formatImageFileName);
             File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
-            logger.info(String.format("Saving to %s", destinationFolder.getPath()));
+            logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
+            logger.debug(String.format("Creating %s", destinationFolder.getPath()));
+            destinationFolder.mkdirs();
             if (!destinationFolder.exists()) {
-                logger.info(String.format("Creating %s", destinationFolder.getPath()));
-                destinationFolder.mkdirs();
+                logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
             }
 
             HttpClient client = cm.getClient().build();
 
-            logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
+            logger.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
+            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest(), context)) {
 
-                if(response.getStatusLine().getStatusCode() / 100 != 2) {
+                if (response.getStatusLine().getStatusCode() / 100 != 2) {
                     EntityUtils.consumeQuietly(response.getEntity());
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
-                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
-                InputStream downloadStream = response.getEntity().getContent();
-                FileOutputStream fos = new FileOutputStream(outputFile);
-                try {
+                File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
+                try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
                     image.setTotal(response.getEntity().getContentLength());
-                    logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
-                    logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
+                    logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
+                    logger.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
 
                     byte[] buffer = new byte[READ_BUFFER_SIZE];
                     int read;
@@ -120,77 +133,71 @@ abstract public class Host {
                         image.increase(read);
                         downloadSpeedService.increase(read);
                     }
+                    fos.flush();
                     EntityUtils.consumeQuietly(response.getEntity());
-                } finally {
-                    if (downloadStream != null) {
-                        downloadStream.close();
-                    }
-                    if (fos != null) {
-                        fos.close();
-                    }
                 }
-                checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
+                File finalName = checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
+                imageFileData.setFileName(finalName.getName());
             }
         } catch (Exception e) {
-            if(Thread.interrupted()) {
+            if (Thread.interrupted()) {
                 throw new InterruptedException("Download was interrupted");
             }
             throw new DownloadException(e);
         }
     }
 
-    private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
-        String formatName;
+    private File checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
         try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
             Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
             if (!it.hasNext()) {
                 throw new HostException("Image file is not recognized!");
             }
             ImageReader reader = it.next();
-            formatName = reader.getFormatName();
-            if (formatName.toUpperCase().equals("JPEG")) {
-                formatName = "jpg";
+            if (reader.getFormatName().toUpperCase().equals("JPEG")) {
+                String imageNameLC = imageName.toLowerCase();
+                if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
+                    imageName += ".jpg";
+                } else {
+                    String toReplace = null;
+                    if (imageNameLC.endsWith("_jpg")) {
+                        toReplace = "_jpg";
+                    } else if (imageNameLC.endsWith("_jpeg")) {
+                        toReplace = "_jpeg";
+                    }
+                    if (toReplace != null) {
+                        imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
+                    }
+                }
             }
         } catch (Exception e) {
             throw new HostException("Failed to guess image format", e);
         }
         try {
-            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
-            if (outImage.exists()) {
-                outImage.delete();
+            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName);
+            if (outImage.exists() && outImage.delete()) {
+                logger.debug(String.format("%s is deleted", outImage.toString()));
             }
-            Files.move(outputFile.toPath(), outImage.toPath());
+            return Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
         } catch (Exception e) {
             throw new HostException("Failed to rename the image", e);
         }
     }
 
-    /**
-     * Just for testing, you may ignore
-     * @throws Exception
-     */
-    private void randomFail() throws Exception {
-        Random random = new Random();
-        int i = random.nextInt(100);
-        if (i < 50) {
-            throw new Exception("Purpose error");
-        }
-    }
-
-    protected final String getDefaultImageName(final String imgUrl) {
-        String imageTitle  = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
+    final String getDefaultImageName(final String imgUrl) {
+        String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
         logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
         return imgUrl;
     }
 
-    protected final Response getResponse(final String url) throws HostException {
+    final Response getResponse(final String url, final HttpClientContext context) throws HostException {
         String basePage;
 
         HttpClient client = cm.getClient().build();
         HttpGet httpGet = cm.buildHttpGet(url);
         Header[] headers;
-        logger.info(String.format("Requesting %s", url));
-        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
+        logger.debug(String.format("Requesting %s", url));
+        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
             if (response.getStatusLine().getStatusCode() / 100 != 2) {
                 throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
             }
@@ -203,27 +210,51 @@ abstract public class Host {
         }
 
         try {
-            logger.info(String.format("Cleaning %s response", url));
+            logger.debug(String.format("Cleaning %s response", url));
             return new Response(htmlProcessorService.clean(basePage), headers);
         } catch (HtmlProcessorException e) {
             throw new HostException(e);
         }
     }
 
-    protected void setImageRequest(final ImageFileData imageFileData) {
+    protected String appendUri(String uri, String appendQuery) throws URISyntaxException {
+        URI oldUri = new URI(uri);
+
+        String newQuery = oldUri.getQuery();
+        if (newQuery == null) {
+            newQuery = appendQuery;
+        } else {
+            newQuery += "&" + appendQuery;
+        }
+
+        return new URI(oldUri.getScheme(), oldUri.getAuthority(),
+                oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
+    }
+
+    private void setImageRequest(final ImageFileData imageFileData) {
         HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
         httpGet.addHeader("Referer", imageFileData.getPageUrl());
         imageFileData.setImageRequest(httpGet);
     }
 
-    protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
+    protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Host host = (Host) o;
+        return Objects.equals(getHost(), host.getHost());
+    }
 
     @Getter
     public static class Response {
+
         protected Response(Document document, Header[] headers) {
             this.document = document;
             this.headers = headers;
         }
+
         private Document document;
         private Header[] headers;
     }

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageBamHost.java

@@ -1,9 +1,9 @@
 package tn.mnlr.vripper.host;
 
-import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -18,8 +18,6 @@ import tn.mnlr.vripper.q.ImageFileData;
 import tn.mnlr.vripper.services.ConnectionManager;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.stream.Collectors;
 
 @Service
 public class ImageBamHost extends Host {
@@ -27,8 +25,8 @@ public class ImageBamHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
 
     private static final String host = "imagebam.com";
-    public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
-    public static final String IMG_XPATH = "//img[@class='image']";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@class='image']";
 
     @Autowired
     private ConnectionManager cm;
@@ -39,34 +37,31 @@ public class ImageBamHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Response response = getResponse(url);
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Response response = getResponse(url, context);
         Document doc = response.getDocument();
-        Header[] headers = response.getHeaders();
-        String cookies = Arrays.asList(headers)
-                .stream()
-                .filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
-                .map(e -> e.getValue().split(";")[0].trim())
-                .collect(Collectors.joining("; "));
 
         Node contDiv;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
-            logger.info(String.format("Getting cookies to use for %s", url));
-            logger.info(String.format("Click button found for %s", url));
+            logger.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpGet httpGet = cm.buildHttpGet(url);
             httpGet.addHeader("Referer", url);
-            httpGet.addHeader("Cookie", cookies);
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
+            logger.debug(String.format("Requesting %s", httpGet));
+            try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
                 String s = EntityUtils.toString(res.getEntity());
                 logger.debug(String.format("%s response is:%n%s", httpGet, s));
                 logger.debug(String.format("Cleaning response for %s", httpGet));
@@ -79,14 +74,14 @@ public class ImageBamHost extends Host {
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageTwistHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -16,7 +17,7 @@ import java.util.Optional;
 @Service
 public class ImageTwistHost extends Host {
 
-    public static final String IMG_XPATH = "//img[contains(@class, 'img')]";
+    private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
     private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
     private static final String host = "imagetwist.com";
     @Autowired
@@ -28,20 +29,25 @@ public class ImageTwistHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageVenueHost.java

@@ -0,0 +1,79 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.net.URI;
+
+@Service
+public class ImageVenueHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImageVenueHost.class);
+
+    private static final String host = "imagevenue.com";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@id='thepic']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        //Sadly, they do not support https.
+        //If they add such support in the future,
+        //then we should automatically adapt the URL here as done elsewhere.
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            if(xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
+                //Button detected. No need to actually click it, just make the call again.
+                resp = getResponse(url, context);
+                doc = resp.getDocument();
+            }
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        if (imgNode == null) {
+            throw new HostException("Failed to locate image");
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            URI baseUri = new URI(url);
+            imageFileData.setImageUrl(new URI(baseUri.getScheme(), baseUri.getHost(), '/'+imgUrl, null).toString());
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch(Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +18,8 @@ public class ImageZillaHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
 
     private static final String host = "imagezilla.net";
-    public static final String IMG_XPATH = "//img[@id='photo']";
+    private static final String lookup = "imagezilla.net/show";
+    private static final String IMG_XPATH = "//img[@id='photo']";
 
     @Autowired
     private ConnectionManager cm;
@@ -28,15 +30,20 @@ public class ImageZillaHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return lookup;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         String title;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
-            logger.info(String.format("Resolving name for %s", url));
+            logger.debug(String.format("Resolving name for %s", url));
             if(titleNode != null) {
                 title = titleNode.getTextContent().trim();
             } else {

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgSpiceHost.java

@@ -0,0 +1,62 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class ImgSpiceHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImgSpiceHost.class);
+
+    private static final String host = "imgspice.com";
+    private static final String IMG_XPATH = "//img[@id='imgpreview']";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +18,7 @@ public class ImgboxHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
 
     private static final String host = "imgbox.com";
-    public static final String IMG_XPATH = "//img[@id='img']";
+    private static final String IMG_XPATH = "//img[@id='img']";
 
     @Autowired
     private ConnectionManager cm;
@@ -28,20 +29,25 @@ public class ImgboxHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -29,8 +30,8 @@ public class ImxHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
 
     private static final String host = "imx.to";
-    public static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
-    public static final String IMG_XPATH = "//img[@class='centred']";
+    private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
+    private static final String IMG_XPATH = "//img[@class='centred']";
 
     @Autowired
     private ConnectionManager cm;
@@ -41,16 +42,21 @@ public class ImxHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
 
         String url = _url.replace("http://", "https://");
-        Response resp = getResponse(url);
+        Response resp = getResponse(url, context);
         Document doc = resp.getDocument();
 
         Node contDiv;
         String value = null;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
             Node node = contDiv.getAttributes().getNamedItem("value");
             if (node != null) {
@@ -60,40 +66,38 @@ public class ImxHost extends Host {
             throw new HostException(e);
         }
 
-        if (contDiv != null) {
-            if (value == null) {
-                throw new HostException("Failed to obtain value attribute from continue input");
-            }
-            logger.info(String.format("Click button found for %s", url));
-            HttpClient client = cm.getClient().build();
-            HttpPost httpPost = cm.buildHttpPost(url);
-            List<NameValuePair> params = new ArrayList<>();
-            params.add(new BasicNameValuePair("imgContinue", value));
-            try {
-                httpPost.setEntity(new UrlEncodedFormEntity(params));
-            } catch (Exception e) {
-                throw new HostException(e);
-            }
-            logger.info(String.format("Requesting %s", httpPost));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
-                logger.debug(String.format("Cleaning response for %s", httpPost));
-                doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (IOException | HtmlProcessorException e) {
-                throw new HostException(e);
-            }
+        if (value == null) {
+            throw new HostException("Failed to obtain value attribute from continue input");
+        }
+        logger.debug(String.format("Click button found for %s", url));
+        HttpClient client = cm.getClient().build();
+        HttpPost httpPost = cm.buildHttpPost(url);
+        List<NameValuePair> params = new ArrayList<>();
+        params.add(new BasicNameValuePair("imgContinue", value));
+        try {
+            httpPost.setEntity(new UrlEncodedFormEntity(params));
+        } catch (Exception e) {
+            throw new HostException(e);
+        }
+        logger.debug(String.format("Requesting %s", httpPost));
+        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
+            logger.debug(String.format("Cleaning response for %s", httpPost));
+            doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
+            EntityUtils.consumeQuietly(response.getEntity());
+        } catch (IOException | HtmlProcessorException e) {
+            throw new HostException(e);
         }
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/PimpandhostHost.java

@@ -0,0 +1,68 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class PimpandhostHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PimpandhostHost.class);
+
+    private static final String host = "pimpandhost.com";
+    private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url;
+        try {
+            url = appendUri(_url.replace("http://", "https://"), "size=original");
+        } catch (Exception e) {
+            throw new HostException(e);
+        }
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +18,8 @@ public class PixhostHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
 
     private static final String host = "pixhost.to";
-    public static final String IMG_XPATH = "//img[@id='image']";
+    private static final String lookup = "pixhost.to/show";
+    private static final String IMG_XPATH = "//img[@id='image']";
 
     @Autowired
     private ConnectionManager cm;
@@ -28,20 +30,25 @@ public class PixhostHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return lookup;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/PixxxelsHost.java

@@ -0,0 +1,67 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class PixxxelsHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PixxxelsHost.class);
+
+    private static final String host = "pixxxels.cc";
+    private static final String IMG_XPATH = "//*[@id='download']";
+    private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = titleNode.getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PostImgHost.java

@@ -0,0 +1,61 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.util.Optional;
+
+@Service
+public class PostImgHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PostImgHost.class);
+
+    private static final String host = "postimg.cc";
+    private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
+    private static final String IMG_XPATH = "//a[@id='download']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Document doc = getResponse(url, context).getDocument();
+
+        Node urlNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            urlNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> getDefaultImageName(url));
+
+            imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
+            imageFileData.setImageName(imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/TurboImageHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,8 +18,8 @@ public class TurboImageHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
 
     private static final String host = "turboimagehost.com";
-    public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
-    public static final String IMG_XPATH = "//img[@id='uImage']";
+    private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
+    private static final String IMG_XPATH = "//img[@id='uImage']";
 
     @Autowired
     private ConnectionManager cm;
@@ -29,15 +30,20 @@ public class TurboImageHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         String title;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
             Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
-            logger.info(String.format("Resolving name for %s", url));
+            logger.debug(String.format("Resolving name for %s", url));
             if(titleNode != null) {
                 title = titleNode.getTextContent().trim();
             } else {

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java

@@ -18,14 +18,14 @@ public class DownloadJob implements Callable<Image> {
     @Getter
     private final ImageFileData imageFileData = new ImageFileData();
 
-    public DownloadJob(Image image) {
+    DownloadJob(Image image) {
         this.image = image;
     }
 
     @Override
     public Image call() throws Exception {
 
-        logger.info(String.format("Starting downloading %s", image.getUrl()));
+        logger.debug(String.format("Starting downloading %s", image.getUrl()));
         image.setStatus(Image.Status.DOWNLOADING);
         image.setCurrent(0);
         image.getHost().download(image, imageFileData);

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java

@@ -1,156 +1,83 @@
 package tn.mnlr.vripper.q;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
 
-import java.util.Arrays;
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
+import java.util.Map;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingDeque;
 
 @Service
 public class DownloadQ {
 
-    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
-
     private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
 
     @Autowired
     private AppStateService appStateService;
 
+    private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> downloadQ = new ConcurrentHashMap<>();
     @Autowired
-    private ExecutionService executionService;
+    private AppSettingsService appSettingsService;
+    @Autowired
+    private List<Host> hosts;
 
-    private BlockingQueue<DownloadJob> downloadQ = new LinkedBlockingQueue<>();
-
-    @Getter
-    private boolean notPauseQ = true;
+    @PostConstruct
+    private void init() {
+        hosts.forEach(host -> downloadQ.put(host, new LinkedBlockingDeque<>()));
+    }
 
     public void put(Image image) throws InterruptedException {
-        synchronized (appStateService) {
-            logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
-            image.init();
-            DownloadJob downloadJob = new DownloadJob(image);
-            downloadQ.put(downloadJob);
-            appStateService.newDownloadJob(downloadJob);
+        logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
+        image.init();
+        DownloadJob downloadJob = new DownloadJob(image);
+        downloadQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
+        appStateService.newDownloadJob(downloadJob);
+    }
+
+    public void remove(final DownloadJob downloadJob) {
+        downloadQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
+    }
+
+    public List<DownloadJob> peek() {
+        List<DownloadJob> downloadJobs = new ArrayList<>();
+        if (hosts.size() == 0) {
+            return downloadJobs;
         }
+        for (Host host : hosts) {
+            Iterator<DownloadJob> it = downloadQ.get(host).iterator();
+            for (int i = 0; i < appSettingsService.getMaxThreads(); i++) {
+                DownloadJob downloadJob = it.hasNext() ? it.next() : null;
+                if (downloadJob != null) {
+                    downloadJobs.add(downloadJob);
+                }
+            }
+        }
+        return downloadJobs;
     }
 
-    public DownloadJob take() throws InterruptedException {
-        DownloadJob downloadJob = downloadQ.take();
-        logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
-        return downloadJob;
-    }
-
-    public synchronized void enqueue(Post post) throws InterruptedException {
+    public void enqueue(Post post) throws InterruptedException {
         for (Image image : post.getImages()) {
             put(image);
         }
     }
 
-    public void restart(String postId) throws InterruptedException {
-        synchronized (appStateService) {
-            if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
-                logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
-                return;
-            }
-            List<Image> images = appStateService.getPost(postId)
-                    .getImages()
-                    .stream()
-                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                    .collect(Collectors.toList());
-            if (images.isEmpty()) {
-                return;
-            }
-            appStateService.getPost(postId).setStatus(Post.Status.PENDING);
-            logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
-            for (Image image : images) {
-                put(image);
-            }
-        }
-    }
-
-    public void removeScheduled(Image image) {
-        synchronized (appStateService) {
-            image.setStatus(Image.Status.STOPPED);
-            logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
-
-            Iterator<DownloadJob> iterator = downloadQ.iterator();
-            boolean removed = false;
-            while (iterator.hasNext()) {
-                DownloadJob next = iterator.next();
-                if (next.getImage().getPostId().equals(image.getPostId())) {
-                    iterator.remove();
-                    appStateService.doneDownloadJob(image);
-                    logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
-                    removed = true;
-                    break;
-                }
-            }
-
-            if (!removed) {
-                logger.warn(String.format("Job for %s does not exist", image.getUrl()));
-            }
-
-            image.cleanup();
-        }
-    }
-
-    public void removeRunning(String postId) {
-        logger.info(String.format("Interrupting running jobs for post id %s", postId));
-        executionService.stop(postId);
-    }
-
-
-    public void stop(String postId) {
-        try {
-            synchronized (appStateService) {
-                if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
-                    return;
-                }
-                notPauseQ = false;
-                appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
-                List<Image> images = appStateService.getPost(postId)
-                        .getImages()
-                        .stream()
-                        .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                        .collect(Collectors.toList());
-                if (images.isEmpty()) {
-                    return;
-                }
-                logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
-                images.forEach(image -> removeScheduled(image));
-                removeRunning(postId);
-            }
-        } finally {
-            notPauseQ = true;
-        }
-    }
-
     public int size() {
-        return downloadQ.size();
+        return downloadQ.values().stream().mapToInt(BlockingDeque::size).sum();
     }
 
-    public void stopAll() {
-        synchronized (appStateService) {
-            appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
-        }
-    }
-
-    public void restartAll() throws InterruptedException {
-        synchronized (appStateService) {
-            for (Post post : appStateService.getCurrentPosts().values()) {
-                String postId = post.getPostId();
-                restart(postId);
-            }
-        }
+    public Iterable<? extends Map.Entry<Host, BlockingDeque<DownloadJob>>> entries() {
+        return downloadQ.entrySet();
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -7,17 +7,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
+import tn.mnlr.vripper.services.ThumbnailGenerator;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.time.temporal.ChronoUnit;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -27,6 +29,8 @@ public class ExecutionService {
 
     private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
 
+    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
+
     @Autowired
     private DownloadQ downloadQ;
 
@@ -36,29 +40,40 @@ public class ExecutionService {
     @Autowired
     private AppStateService appStateService;
 
-    private AtomicInteger threadCount = new AtomicInteger();
+    @Autowired
+    private ThumbnailGenerator thumbnailGenerator;
 
-    private ExecutorService executor = Executors.newFixedThreadPool(10);
+    @Autowired
+    private AppSettingsService appSettingsService;
+    private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
+    private boolean notPauseQ = true;
+    private ExecutorService executor = Executors.newFixedThreadPool(20);
 
     private Thread executionThread;
 
+    BlockingQueue<DownloadJob> queue = new LinkedBlockingQueue<>();
+
     private RetryPolicy<Object> retryPolicy;
 
     private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
 
     private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
+    private Thread pollThread;
 
     @PostConstruct
     private void init() {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withDelay(Duration.ofSeconds(5))
-                .withMaxRetries(2)
+                .withBackoff(1, 10, ChronoUnit.SECONDS)
+                .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+                .withMaxAttempts(3)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 
         executionThread = new Thread(this::start, "Executor thread");
+        pollThread = new Thread(this::poll, "Polling thread");
+        pollThread.start();
         executionThread.start();
     }
 
@@ -68,92 +83,199 @@ public class ExecutionService {
         executionThread.interrupt();
         executor.shutdown();
         appStateService.getCurrentPosts().keySet().forEach(p -> {
-            logger.info(String.format("Stopping download jobs for %s", p));
-            this.stop(p);
+            logger.debug(String.format("Stopping download jobs for %s", p));
+            this.stopRunning(p);
         });
         executor.awaitTermination(10, TimeUnit.SECONDS);
     }
 
-    public void stop(String postId) {
+    public void stopRunning(String postId) {
         List<DownloadJob> data = running
                 .stream()
                 .filter(e -> e.getImage().getPostId().equals(postId))
                 .peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
                 .collect(Collectors.toList());
-        logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
+        logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
 
         data.forEach(e -> {
             futures.get(e.getImage().getUrl()).cancel(true);
-            if(e.getImageFileData().getImageRequest() != null) {
+            if (e.getImageFileData().getImageRequest() != null) {
                 e.getImageFileData().getImageRequest().abort();
             }
             e.getImage().cleanup();
         });
     }
 
-    boolean canRun() {
-        boolean canRun = threadCount.get() < settings.getMaxThreads();
-        if (canRun && downloadQ.isNotPauseQ()) {
-            threadCount.incrementAndGet();
+
+    public synchronized void stopAll() {
+        appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
+    }
+
+    public synchronized void restartAll() throws InterruptedException {
+        for (Post post : appStateService.getCurrentPosts().values()) {
+            String postId = post.getPostId();
+            restart(postId);
+        }
+    }
+
+    public synchronized void restart(String postId) throws InterruptedException {
+        if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
+            logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
+            return;
+        }
+        List<Image> images = appStateService.getPost(postId)
+                .getImages()
+                .stream()
+                .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                .collect(Collectors.toList());
+        if (images.isEmpty()) {
+            return;
+        }
+        appStateService.getPost(postId).setStatus(Post.Status.PENDING);
+        logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
+        for (Image image : images) {
+            downloadQ.put(image);
+        }
+    }
+
+    private synchronized void removeScheduled(Image image) {
+        image.setStatus(Image.Status.STOPPED);
+        logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
+
+        boolean removed = false;
+        main:
+        for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : downloadQ.entries()) {
+            Iterator<DownloadJob> iterator = entry.getValue().iterator();
+            while (iterator.hasNext()) {
+                DownloadJob next = iterator.next();
+                if (next.getImage().getPostId().equals(image.getPostId())) {
+                    iterator.remove();
+                    appStateService.doneDownloadJob(image);
+                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
+                    removed = true;
+                    break main;
+                }
+            }
+        }
+
+        if (!removed) {
+            logger.debug(String.format("Job for %s does not exist", image.getUrl()));
+        }
+
+        image.cleanup();
+    }
+
+    private synchronized void removeRunning(String postId) {
+        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
+        stopRunning(postId);
+    }
+
+
+    public synchronized void stop(String postId) {
+        try {
+            if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
+                return;
+            }
+            notPauseQ = false;
+            appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
+            List<Image> images = appStateService.getPost(postId)
+                    .getImages()
+                    .stream()
+                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                    .collect(Collectors.toList());
+            if (images.isEmpty()) {
+                return;
+            }
+            logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
+            images.forEach(this::removeScheduled);
+            removeRunning(postId);
+        } finally {
+            notPauseQ = true;
+        }
+    }
+
+    private boolean canRun(Host host) {
+        boolean canRun;
+        AtomicInteger count = threadCount.get(host);
+        if (count == null) {
+            threadCount.put(host, new AtomicInteger(0));
+        }
+        canRun = threadCount.get(host).get() < settings.getMaxThreads() && threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getMaxTotalThreads();
+        if (canRun && notPauseQ) {
+            threadCount.get(host).incrementAndGet();
             return true;
         }
         return false;
     }
 
-    public void start() {
+    public void poll() {
         while (!Thread.interrupted()) {
-            if (canRun()) {
-                DownloadJob take = null;
-                try {
-                    take = downloadQ.take();
-                    if (take == null) {
-                        continue;
+            try {
+                List<DownloadJob> peek = downloadQ.peek();
+                for (DownloadJob downloadJob : peek) {
+                    if (canRun(downloadJob.getImage().getHost())) {
+                        queue.offer(downloadJob);
+                        downloadQ.remove(downloadJob);
                     }
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    break;
                 }
-
-                DownloadJob finalTake = take;
-                Runnable task = () -> {
-                    running.add(finalTake);
-
-                    Failsafe.with(retryPolicy)
-                            .onFailure(e -> {
-                                if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
-                                    logger.info("Job successfully interrupted");
-                                    return;
-                                }
-                                logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
-                                finalTake.getImage().setStatus(Image.Status.ERROR);
-                            })
-                            .onComplete(e -> {
-                                appStateService.doneDownloadJob(finalTake.getImage());
-                                logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
-                                synchronized (threadCount) {
-                                    threadCount.decrementAndGet();
-                                    running.remove(finalTake);
-                                    futures.remove(finalTake.getImage().getUrl());
-                                    threadCount.notify();
-                                }
-                            })
-                            .get(finalTake::call);
-                };
-                logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
-                futures.put(finalTake.getImage().getUrl(), executor.submit(task));
-            } else {
                 synchronized (threadCount) {
-                    try {
-                        threadCount.wait(2_000);
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                        break;
-                    }
+                    threadCount.wait(2_000);
                 }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
             }
         }
     }
 
+    public void start() {
+        while (!Thread.interrupted()) {
+            try {
+                push(queue.take());
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            } catch (Exception e) {
+                logger.error("Execution Service failed", e);
+                break;
+            }
+        }
+    }
+
+    private void push(DownloadJob take) {
+        Runnable task = () -> {
+            running.add(take);
+
+            Failsafe.with(retryPolicy)
+                    .onFailure(e -> {
+                        if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
+                            logger.debug("Job successfully interrupted");
+                            return;
+                        }
+                        logger.error(String.format("Failed to download %s after %d tries", take.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
+                        take.getImage().setStatus(Image.Status.ERROR);
+                    })
+                    .onComplete(e -> {
+                        appStateService.doneDownloadJob(take.getImage());
+                        logger.debug(String.format("Finished downloading %s", take.getImage().getUrl()));
+                        if (appSettingsService.isViewPhotos()) {
+                            VripperApplication.commonExecutor.submit(
+                                    () -> thumbnailGenerator.getThumbnails()
+                                            .get(new ThumbnailGenerator.CacheKey(take.getImage().getPostId(), take.getImageFileData().getFileName())));
+                        }
+                        threadCount.get(take.getImage().getHost()).decrementAndGet();
+                        running.remove(take);
+                        futures.remove(take.getImage().getUrl());
+                        synchronized (threadCount) {
+                            threadCount.notify();
+                        }
+                    })
+                    .get(take::call);
+        };
+        logger.debug(String.format("Scheduling a job for %s", take.getImage().getUrl()));
+        futures.put(take.getImage().getUrl(), executor.submit(task));
+    }
+
     public int runningCount() {
         return running.size();
     }

vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java

@@ -10,6 +10,7 @@ public class ImageFileData {
 
     private String pageUrl;
     private String imageName;
+    private String fileName;
     private String imageUrl;
     private HttpUriRequest imageRequest;
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -7,9 +7,11 @@ import lombok.Setter;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.exception.ValidationException;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
@@ -23,12 +25,18 @@ import java.util.prefs.Preferences;
 @Setter
 public class AppSettingsService {
 
+    private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
+
+    @Value("${base.dir}")
+    private String defaultDownloadPath;
+
     private static final Logger logger = LoggerFactory.getLogger(AppSettingsService.class);
 
     private Preferences prefs = Preferences.userNodeForPackage(AppSettingsService.class);
 
     private final String DOWNLOAD_PATH = "DOWNLOAD_PATH";
     private final String MAX_THREADS = "MAX_THREADS";
+    private int maxTotalThreads;
     private final String AUTO_START = "AUTO_START";
     private final String V_LOGIN = "VLOGIN";
     private final String V_USERNAME = "VUSERNAME";
@@ -40,9 +48,16 @@ public class AppSettingsService {
     private final String THREADSUBFOLDER = "THREADSUBFOLDER";
     private final String CLEAR = "CLEAR";
     private final String DARK_THEME = "DARK_THEME";
+    private final String VIEW_PHOTOS = "VIEW_PHOTOS";
 
     private String downloadPath;
     private int maxThreads;
+
+    @PostConstruct
+    private void init() {
+        restore();
+    }
+
     private boolean autoStart;
     private boolean vLogin;
     private String vUsername;
@@ -54,6 +69,7 @@ public class AppSettingsService {
     private boolean forceOrder;
     private boolean clearCompleted;
     private boolean darkTheme;
+    private boolean viewPhotos;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -65,8 +81,9 @@ public class AppSettingsService {
 
     public void restore() {
 
-        downloadPath = prefs.get(DOWNLOAD_PATH, System.getProperty("user.dir"));
+        downloadPath = prefs.get(DOWNLOAD_PATH, defaultDownloadPath);
         maxThreads = prefs.getInt(MAX_THREADS, 4);
+        maxTotalThreads = prefs.getInt(MAX_TOTAL_THREADS, 8);
         autoStart = prefs.getBoolean(AUTO_START, true);
         vLogin = prefs.getBoolean(V_LOGIN, false);
         vUsername = prefs.get(V_USERNAME, "");
@@ -78,6 +95,7 @@ public class AppSettingsService {
         threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
         clearCompleted = prefs.getBoolean(CLEAR, false);
         darkTheme = prefs.getBoolean(DARK_THEME, false);
+        viewPhotos = prefs.getBoolean(VIEW_PHOTOS, false);
     }
 
     @PreDestroy
@@ -85,6 +103,7 @@ public class AppSettingsService {
 
         prefs.put(DOWNLOAD_PATH, downloadPath);
         prefs.putInt(MAX_THREADS, maxThreads);
+        prefs.putInt(MAX_TOTAL_THREADS, maxTotalThreads);
         prefs.putBoolean(AUTO_START, autoStart);
         prefs.putBoolean(V_LOGIN, vLogin);
         prefs.put(V_USERNAME, vUsername);
@@ -96,6 +115,7 @@ public class AppSettingsService {
         prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
         prefs.putBoolean(CLEAR, clearCompleted);
         prefs.putBoolean(DARK_THEME, darkTheme);
+        prefs.putBoolean(VIEW_PHOTOS, viewPhotos);
 
         try {
             prefs.sync();
@@ -118,8 +138,12 @@ public class AppSettingsService {
             throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
         }
 
-        if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 8) {
-            throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 8));
+        if (settings.getMaxTotalThreads() < 1) {
+            throw new ValidationException(String.format("Invalid max global concurrent download settings, values must be in greater than %d", 1));
+        }
+
+        if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
+            throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
         }
     }
 
@@ -139,7 +163,7 @@ public class AppSettingsService {
         @JsonProperty("darkTheme")
         private boolean darkTheme;
 
-        public Theme(boolean darkTheme) {
+        Theme(boolean darkTheme) {
             this.darkTheme = darkTheme;
         }
     }
@@ -151,6 +175,8 @@ public class AppSettingsService {
         private String downloadPath;
         @JsonProperty("maxThreads")
         private int maxThreads;
+        @JsonProperty("maxTotalThreads")
+        private int maxTotalThreads;
         @JsonProperty("autoStart")
         private boolean autoStart;
         @JsonProperty("vLogin")
@@ -171,10 +197,13 @@ public class AppSettingsService {
         private boolean threadSubLocation;
         @JsonProperty("clearCompleted")
         private boolean clearCompleted;
+        @JsonProperty("viewPhotos")
+        private boolean viewPhotos;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted) {
+        public Settings(String downloadPath, int maxThreads, int maxTotalThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted, boolean viewPhotos) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
+            this.maxTotalThreads = maxTotalThreads;
             this.autoStart = autoStart;
             this.vLogin = vLogin;
             this.vUsername = vUsername;
@@ -185,6 +214,7 @@ public class AppSettingsService {
             this.subLocation = subLocation;
             this.threadSubLocation = threadSubLocation;
             this.clearCompleted = clearCompleted;
+            this.viewPhotos = viewPhotos;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -1,6 +1,5 @@
 package tn.mnlr.vripper.services;
 
-import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.PublishProcessor;
 import lombok.Getter;
 import lombok.Setter;
@@ -12,7 +11,6 @@ import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.q.DownloadJob;
 
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -31,6 +29,10 @@ public class AppStateService {
 
     private Map<String, AtomicInteger> runningPosts = new ConcurrentHashMap<>();
 
+    private Map<String, QueuedVGLink> grabQueue = new ConcurrentHashMap<>();
+
+    private PublishProcessor<QueuedVGLink> liveGrabQueue = PublishProcessor.create();
+
     private PublishProcessor<Image> liveImageUpdates = PublishProcessor.create();
 
     private PublishProcessor<Post> livePostsState = PublishProcessor.create();
@@ -41,8 +43,6 @@ public class AppStateService {
     @Autowired
     private AppSettingsService appSettingsService;
 
-    private Disposable subscription;
-
     public void onImageUpdate(Image imageState) {
 
         persistenceService.getProcessor().onNext(currentPosts);
@@ -59,7 +59,7 @@ public class AppStateService {
         return currentPosts.get(postId);
     }
 
-    public synchronized void newDownloadJob(DownloadJob downloadJob) {
+    public void newDownloadJob(DownloadJob downloadJob) {
         String postId = downloadJob.getImage().getPostId();
         checkKeyRunningPosts(postId);
         runningPosts.get(postId).incrementAndGet();
@@ -67,7 +67,7 @@ public class AppStateService {
 
     public synchronized void postDownloadingUpdate(String postId) {
         Post post = currentPosts.get(postId);
-        if (!post.getStatus().equals(Post.Status.DOWNLOADING)) {
+        if (!post.getStatus().equals(Post.Status.DOWNLOADING) && !post.getStatus().equals(Post.Status.PARTIAL)) {
             post.setStatus(Post.Status.DOWNLOADING);
             livePostsState.onNext(post);
         }
@@ -81,7 +81,7 @@ public class AppStateService {
             post.setStatus(Post.Status.PARTIAL);
         }
         if (i == 0) {
-            if (post.getImages().stream().map(Image::getStatus).filter(e -> e.equals(Image.Status.ERROR)).count() > 0) {
+            if (post.getImages().stream().map(Image::getStatus).anyMatch(e -> e.equals(Image.Status.ERROR))) {
                 post.setStatus(Post.Status.ERROR);
             } else {
                 if (!Post.Status.STOPPED.equals(post.getStatus())) {
@@ -104,13 +104,7 @@ public class AppStateService {
         currentPosts.get(postId).setRemoved(true);
         runningPosts.remove(postId);
         currentPosts.remove(postId);
-        Iterator<Map.Entry<String, Image>> iterator = currentImages.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, Image> entry = iterator.next();
-            if(entry.getValue().getPostId().equals(postId)) {
-                iterator.remove();
-            }
-        }
+        currentImages.entrySet().removeIf(entry -> entry.getValue().getPostId().equals(postId));
         persistenceService.getProcessor().onNext(currentPosts);
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -4,16 +4,19 @@ import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.client.LaxRedirectStrategy;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
 import java.net.URI;
+import java.util.concurrent.TimeUnit;
 
 @Service
+@EnableScheduling
 public class ConnectionManager {
 
     private ConnectionManager() {
@@ -23,35 +26,41 @@ public class ConnectionManager {
     private PoolingHttpClientConnectionManager pcm;
 
     private RequestConfig rc = RequestConfig.custom()
-            .setConnectionRequestTimeout(10_000)
-            .setConnectTimeout(10_000)
-            .setSocketTimeout(10_000)
+            .setConnectionRequestTimeout(5_000)
+            .setConnectTimeout(5_000)
+            .setSocketTimeout(5_000)
             .setCookieSpec(CookieSpecs.STANDARD)
             .build();
 
     private void buildConnectionPool() {
 
         pcm = new PoolingHttpClientConnectionManager();
-        pcm.setMaxTotal(200);
-        pcm.setDefaultMaxPerRoute(10);
+        pcm.setMaxTotal(50);
+        pcm.setDefaultMaxPerRoute(4);
+    }
+
+    @Scheduled(fixedDelay = 1000)
+    private void idleConnectionMonitoring() {
+        pcm.closeExpiredConnections();
+        pcm.closeIdleConnections(30, TimeUnit.SECONDS);
     }
 
     public HttpClientBuilder getClient() {
         return HttpClients.custom()
                 .setConnectionManager(pcm)
                 .setRedirectStrategy(new LaxRedirectStrategy())
-                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
+                .disableAutomaticRetries()
                 .setDefaultRequestConfig(rc);
     }
 
     public HttpGet buildHttpGet(String url) {
-        HttpGet httpGet = new HttpGet(url);
+        HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
         httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpGet;
     }
 
     public HttpPost buildHttpPost(String url) {
-        HttpPost httpPost = new HttpPost(url);
+        HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
         httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpPost;
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeed.java

@@ -13,7 +13,7 @@ public class DownloadSpeed {
     }
 
     private String formatSI(long bytes) {
-        return humanReadableByteCount(bytes, true);
+        return humanReadableByteCount(bytes, false);
     }
 
     private String humanReadableByteCount(long bytes, boolean si) {

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalState.java

@@ -13,7 +13,7 @@ public class GlobalState {
     private long remaining;
     private long error;
 
-    public GlobalState(long running, long queued, long remaining, long error) {
+    GlobalState(long running, long queued, long remaining, long error) {
         this.running = running;
         this.queued = queued;
         this.remaining = remaining;
@@ -25,11 +25,7 @@ public class GlobalState {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         GlobalState that = (GlobalState) o;
-        return running == that.running &&
-                queued == that.queued &&
-                remaining == that.remaining &&
-                error == that.error &&
-                Objects.equals(type, that.type);
+        return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
     }
 
     @Override

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java

@@ -29,6 +29,9 @@ public class GlobalStateService {
     @Getter
     private PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
 
+    public GlobalStateService() {
+    }
+
     @Scheduled(fixedDelay = 3000)
     private void interval() {
         GlobalState newGlobalState = new GlobalState(

vripper-server/src/main/java/tn/mnlr/vripper/services/HtmlProcessorService.java

@@ -11,10 +11,6 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
 @Service
 public class HtmlProcessorService {
 
-    private HtmlProcessorService() {
-
-    }
-
     public Document clean(String htmlContent) throws HtmlProcessorException {
         try {
             TagNode clean = new HtmlCleaner().clean(htmlContent);

vripper-server/src/main/java/tn/mnlr/vripper/services/PathService.java

@@ -37,17 +37,10 @@ public class PathService {
     /**
      * Will sanitize the image name and remove extension
      *
-     * @param imageName
-     * @return
+     * @param imageName path string
+     * @return Sanitized local path string
      */
     public final String formatImageFileName(String imageName) {
-        int extensionIndex = imageName.lastIndexOf('.');
-        String fileName;
-        if (extensionIndex != -1) {
-            fileName = imageName.substring(0, extensionIndex);
-        } else {
-            fileName = imageName;
-        }
-        return sanitize(fileName);
+        return sanitize(imageName);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java

@@ -7,9 +7,10 @@ import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.EventListenerBean;
 import tn.mnlr.vripper.SpringContext;
-import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
@@ -29,7 +30,6 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @Service
 public class PersistenceService {
@@ -39,8 +39,11 @@ public class PersistenceService {
     @Autowired
     private AppStateService stateService;
 
-    @Autowired
-    private VripperApplication.AppCommandRunner appCommandRunner;
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Getter
+    private String dataPath;
 
     private ObjectMapper om;
 
@@ -55,31 +58,32 @@ public class PersistenceService {
         om.addMixIn(Post.class, PostPersistanceMixin.class);
         subscription = processor
                 .onBackpressureLatest()
-                .buffer(10, TimeUnit.SECONDS)
-                .filter(e -> !e.isEmpty())
-                .map(e -> e.get(0))
+                .throttleLatest(1, TimeUnit.SECONDS)
                 .doOnNext(this::persist)
                 .subscribe();
     }
 
     @PostConstruct
     public void init() {
-        File dataFile = new File(appCommandRunner.getDataPath());
+        dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
+        File dataFile = new File(dataPath);
         if (!dataFile.exists()) {
             try {
-                dataFile.getParentFile().mkdirs();
+                if (dataFile.getParentFile().mkdirs()) {
+                    logger.debug(String.format("%s is created", dataFile.getParentFile().toString()));
+                }
                 if (!dataFile.getParentFile().isDirectory() || !dataFile.getParentFile().canWrite()) {
                     logger.error(String.format("Unable to write in %s", dataFile.getParent()));
                     SpringContext.close();
                 }
 
                 if (dataFile.createNewFile()) {
-                    logger.info("Data file successfully created");
+                    logger.debug("Data file successfully created");
                     try (FileWriter fw = new FileWriter(dataFile)) {
                         fw.write("{}");
                     }
                 } else {
-                    logger.info("Data file already exists");
+                    logger.warn("Data file already exists");
                 }
             } catch (IOException e) {
                 logger.error("Unable to create data file", e);
@@ -93,12 +97,14 @@ public class PersistenceService {
         this.subscription.dispose();
         logger.info(String.format("Destroying %s", PersistenceService.class.getSimpleName()));
         logger.info("Persisting data before destroying");
-        this.persist(stateService.getCurrentPosts());
+        if (EventListenerBean.isInit()) {
+            this.persist(stateService.getCurrentPosts());
+        }
     }
 
-    public void persist(Map<String, Post> currentPosts) {
+    private void persist(Map<String, Post> currentPosts) {
 
-        try (PrintWriter out = new PrintWriter(appCommandRunner.getDataPath(), "UTF-8")) {
+        try (PrintWriter out = new PrintWriter(dataPath, StandardCharsets.UTF_8)) {
             out.print(om.writeValueAsString(currentPosts));
         } catch (IOException e) {
             logger.error("Failed to persist app state", e);
@@ -112,11 +118,11 @@ public class PersistenceService {
         } catch (IOException e) {
             logger.error("Failed to read app state", e);
             long timestamp = new Date().getTime();
-            logger.warn(String.format("trying to rename old data file from %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + "." + timestamp + ".old"));
+            logger.warn(String.format("trying to rename old data file from %s to %s", dataPath, dataPath + "." + timestamp + ".old"));
             try {
-                Files.move(new File(appCommandRunner.getDataPath()).toPath(), new File(appCommandRunner.getDataPath() + "." + timestamp + ".old").toPath());
+                Files.move(new File(dataPath).toPath(), new File(dataPath + "." + timestamp + ".old").toPath());
             } catch (IOException ex) {
-                logger.error(String.format("Failed to rename %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + ".old"));
+                logger.error(String.format("Failed to rename %s to %s", dataPath, dataPath + ".old"));
                 SpringContext.close();
             }
         }
@@ -128,7 +134,7 @@ public class PersistenceService {
 
         String jsonContent = null;
         try {
-            jsonContent = Files.readAllLines(Paths.get(appCommandRunner.getDataPath()), StandardCharsets.UTF_8).stream().collect(Collectors.joining());
+            jsonContent = String.join("", Files.readAllLines(Paths.get(dataPath), StandardCharsets.UTF_8));
         } catch (Exception e) {
             logger.error("data file cannot be read, previous state cannot be restored", e);
             SpringContext.close();
@@ -139,7 +145,7 @@ public class PersistenceService {
         stateService.getCurrentPosts().clear();
         stateService.getCurrentPosts().putAll(read);
         stateService.getCurrentPosts().values().forEach(p -> {
-            if(Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL).contains(p.getStatus())) {
+            if (Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL, Post.Status.PENDING).contains(p.getStatus())) {
                 p.setStatus(Post.Status.STOPPED);
             }
         });
@@ -150,8 +156,6 @@ public class PersistenceService {
             stateService.getCurrentImages().put(e.getUrl(), e);
         });
 
-        read.values().forEach(e -> {
-            e.setAppStateService(stateService);
-        });
+        read.values().forEach(e -> e.setAppStateService(stateService));
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -1,50 +1,27 @@
 package tn.mnlr.vripper.services;
 
-import io.reactivex.processors.PublishProcessor;
-import lombok.Getter;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
-import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.q.DownloadQ;
 
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.parsers.SAXParserFactory;
-import java.io.BufferedInputStream;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
+import javax.annotation.PostConstruct;
 import java.util.List;
-import java.util.Optional;
 
 @Service
 public class PostParser {
 
     private static final Logger logger = LoggerFactory.getLogger(PostParser.class);
 
-    private static final String VR_API = "https://vipergirls.to/vr.php";
+    static final String VR_API = "https://vipergirls.to/vr.php";
 
     @Autowired
     private ConnectionManager cm;
 
-    SAXParserFactory factory = SAXParserFactory.newInstance();
-
-    @Autowired
-    private List<Host> supportedHosts;
-
     @Autowired
     private AppStateService appStateService;
 
@@ -60,25 +37,31 @@ public class PostParser {
     @Autowired
     private VipergirlsAuthService vipergirlsAuthService;
 
-    public PostParser() throws ParserConfigurationException, SAXException {
+    @Autowired
+    private List<Host> supportedHosts;
+
+    @PostConstruct
+    private void init() {
+
     }
 
-    public void addPost(String postId, String threadId) throws PostParseException {
+    public synchronized void addPost(String postId, String threadId) throws PostParseException {
 
         if (appStateService.getCurrentPosts().containsKey(postId)) {
-            logger.info(String.format("skipping %s, already loaded", postId));
+            logger.warn(String.format("skipping %s, already loaded", postId));
             return;
         }
 
-        VRPostParser vrPostParser = new VRPostParser(threadId, postId);
+        VRPostParser vrPostParser = new VRPostParser(threadId, postId, cm, vipergirlsAuthService, supportedHosts);
         Post post = vrPostParser.parse();
 
         post.setAppStateService(appStateService);
         post.getImages().forEach(e -> e.setAppStateService(appStateService));
         authService.leaveThanks(post.getUrl(), post.getPostId());
         if (appSettingsService.isAutoStart()) {
-            logger.info("Auto start downloads option is enabled");
-            logger.info(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
+            logger.debug("Auto start downloads option is enabled");
+            logger.debug(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
+            post.setStatus(Post.Status.PENDING);
             try {
                 downloadQ.enqueue(post);
             } catch (InterruptedException e) {
@@ -86,307 +69,14 @@ public class PostParser {
                 Thread.currentThread().interrupt();
                 return;
             }
-            logger.info(String.format("Done enqueuing jobs for %s", post.getUrl()));
+            logger.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
         } else {
-            logger.info("Auto start downloads option is disabled");
+            post.setStatus(Post.Status.STOPPED);
+            logger.debug("Auto start downloads option is disabled");
         }
     }
 
-    @Getter
-    public static abstract class VRPostState {
-        private final String threadId;
-
-        protected VRPostState(String threadId) {
-            this.threadId = threadId;
-        }
-    }
-
-    @Getter
-    public static class VRPostParse extends VRPostState {
-
-        private final String type = "postParse";
-        private String postId;
-        private int number;
-        private String title;
-        private int imageCount;
-        private String url;
-        private List<String> previews;
-
-
-        public VRPostParse(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews) {
-            super(threadId);
-            this.postId = postId;
-            this.number = number;
-            this.title = title;
-            this.imageCount = imageCount;
-            this.previews = previews;
-            this.url = url;
-        }
-    }
-
-    @Getter
-    public static class VRThreadParseState extends VRPostState {
-
-        private final String type = "threadParseState";
-        private final String state;
-
-        public VRThreadParseState(String threadId, String state) {
-            super(threadId);
-            this.state = state;
-        }
-    }
-
-    public class VRThreadParser {
-
-        @Getter
-        private final PublishProcessor<VRPostState> postPublishProcessor = PublishProcessor.create();
-
-        private String threadId;
-
-        public VRThreadParser(String threadId) {
-            this.threadId = threadId;
-        }
-
-        public Void parse() throws PostParseException {
-
-            logger.info(String.format("Parsing thread %s", threadId));
-            HttpGet httpGet;
-            try {
-                URIBuilder uriBuilder = new URIBuilder(VR_API);
-                uriBuilder.setParameter("t", threadId);
-                httpGet = cm.buildHttpGet(uriBuilder.build());
-                if (vipergirlsAuthService.getCookies() != null) {
-                    httpGet.addHeader("Cookie", vipergirlsAuthService.getCookies());
-                }
-            } catch (URISyntaxException e) {
-                throw new PostParseException(e);
-            }
-            VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
-            HttpClient connection = cm.getClient().build();
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
-
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
-            }
-            return null;
-        }
-    }
-
-    private class VRThreadHandler extends DefaultHandler {
-
-        private final String threadId;
-        private final PublishProcessor<VRPostState> vrPostPublishProcessor;
-        private String threadTitle;
-        private String postId;
-        private String postTitle;
-        private int imageCount;
-        private int postCounter;
-        private int previewCounter = 0;
-        private List<String> previews = new ArrayList<>();
-
-        public VRThreadHandler(String threadId, PublishProcessor<VRPostState> vrPostPublishProcessor) {
-            this.vrPostPublishProcessor = vrPostPublishProcessor;
-            this.threadId = threadId;
-        }
-
-        @Override
-        public void startDocument() {
-            vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "START"));
-        }
-
-        @Override
-        public void startElement(String uri, String localName, String qName, Attributes attributes) {
-
-            switch (qName.toLowerCase()) {
-                case "thread":
-                    threadTitle = attributes.getValue("title").trim();
-                    break;
-                case "post":
-                    imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
-                    postId = attributes.getValue("id").trim();
-                    postCounter = Integer.parseInt(attributes.getValue("number").trim());
-                    postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
-                    break;
-                case "image":
-                    if (previewCounter++ < 4) {
-                        String thumbUrl = Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).orElse(null);
-                        if (thumbUrl != null) {
-                            previews.add(thumbUrl);
-                        }
-                    }
-                    break;
-            }
-        }
-
-        @Override
-        public void endElement(String uri, String localName, String qName) {
-            switch (qName.toLowerCase()) {
-                case "post":
-                    if (imageCount != 0) {
-                        vrPostPublishProcessor.onNext(new VRPostParse(
-                                threadId,
-                                postId,
-                                postCounter,
-                                postTitle,
-                                imageCount,
-                                String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
-                                previews
-                        ));
-                    }
-                    previewCounter = 0;
-                    previews = new ArrayList<>();
-                    break;
-            }
-        }
-
-        @Override
-        public void endDocument() {
-            vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "END"));
-            vrPostPublishProcessor.onComplete();
-        }
-    }
-
-    public class VRPostParser {
-
-        private String threadId;
-        private String postId;
-
-        public VRPostParser(String threadId, String postId) {
-            this.threadId = threadId;
-            this.postId = postId;
-        }
-
-        public Post parse() throws PostParseException {
-
-            Post post;
-            logger.info(String.format("Parsing post %s", postId));
-            HttpGet httpGet;
-            try {
-                URIBuilder uriBuilder = new URIBuilder(VR_API);
-                uriBuilder.setParameter("p", postId);
-                httpGet = cm.buildHttpGet(uriBuilder.build());
-                if (vipergirlsAuthService.getCookies() != null) {
-                    httpGet.addHeader("Cookie", vipergirlsAuthService.getCookies());
-                }
-            } catch (URISyntaxException e) {
-                throw new PostParseException(e);
-            }
-            VRPostHandler handler = new VRPostHandler(threadId, postId);
-            HttpClient connection = cm.getClient().build();
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
-
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                post = handler.getParsedPost();
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
-            }
-            return post;
-        }
-    }
-
-    private class VRPostHandler extends DefaultHandler {
-
-        private final String threadId;
-        private final String postId;
-        private String threadTitle;
-
-        private int previewCounter = 0;
-        private int index = 0;
-
-        private String postTitle;
-        private int imageCount;
-
-        private String forum;
-
-        private List<String> previews = new ArrayList<>();
-        private List<Image> images = new ArrayList<>();
-        @Getter
-        private Post parsedPost;
-
-        public VRPostHandler(String threadId, String postId) {
-            this.threadId = threadId;
-            this.postId = postId;
-        }
-
-        @Override
-        public void startDocument() {
-        }
-
-        @Override
-        public void startElement(String uri, String localName, String qName, Attributes attributes) {
-
-            switch (qName.toLowerCase()) {
-                case "forum":
-                    forum = attributes.getValue("title").trim();
-                    break;
-                case "thread":
-                    threadTitle = attributes.getValue("title").trim();
-                    break;
-                case "post":
-                    imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
-                    postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
-                    break;
-                case "image":
-                    index++;
-                    if (previewCounter++ < 4) {
-                        String thumbUrl = Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).orElse(null);
-                        if (thumbUrl != null) {
-                            previews.add(thumbUrl);
-                        }
-                    }
-
-                    String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
-                    if (mainUrl != null) {
-                        Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
-                        if (foundHost != null) {
-                            logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
-                            images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
-                        } else {
-                            logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
-                        }
-                    }
-                    break;
-            }
-        }
-
-        @Override
-        public void endElement(String uri, String localName, String qName) {
-            switch (qName.toLowerCase()) {
-                case "post":
-                    if (imageCount != 0) {
-                        HashMap<String, Object> metadata = new HashMap<>();
-                        metadata.put("PREVIEWS", previews);
-                        parsedPost = new Post(
-                                postTitle,
-                                String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
-                                images,
-                                metadata,
-                                postId,
-                                threadId,
-                                threadTitle,
-                                forum
-                        );
-                    }
-                    index = 0;
-                    previewCounter = 0;
-                    previews = new ArrayList<>();
-                    images = new ArrayList<>();
-                    break;
-            }
-        }
+    public VRThreadParser createVRThreadParser(String threadId) {
+        return new VRThreadParser(threadId, cm, vipergirlsAuthService, supportedHosts);
     }
 }
-

vripper-server/src/main/java/tn/mnlr/vripper/services/QueuedVGLink.java

@@ -0,0 +1,38 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.util.Objects;
+
+@Getter
+@ToString
+public class QueuedVGLink {
+
+    private final String type = "grabQueue";
+    private final String link;
+    private final String threadId;
+    private final String postId;
+    @Setter
+    boolean removed;
+
+    public QueuedVGLink(String link, String threadId, String postId) {
+        this.link = link;
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        QueuedVGLink that = (QueuedVGLink) o;
+        return Objects.equals(link, that.link);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(link);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/ThumbnailGenerator.java

@@ -0,0 +1,161 @@
+package tn.mnlr.vripper.services;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.imgscalr.Scalr;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.util.FileSystemUtils;
+
+import javax.annotation.PostConstruct;
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Service
+public class ThumbnailGenerator {
+
+    @Getter
+    private LoadingCache<CacheKey, byte[]> thumbnails;
+
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Autowired
+    private PathService pathService;
+
+    @Getter
+    private File cacheFolder;
+    CacheLoader<CacheKey, byte[]> loader = new CacheLoader<>() {
+        @Override
+        public byte[] load(CacheKey key) throws Exception {
+            File destinationFolder = pathService.getDownloadDestinationFolder(key.getPostId());
+            if (!destinationFolder.exists() || !destinationFolder.isDirectory()) {
+                return null;
+            }
+            return Files.readAllBytes(generateThumbnail(new File(destinationFolder, key.getImgName()), key.getPostId()).toPath());
+        }
+    };
+
+    @PostConstruct
+    private void init() throws Exception {
+        cacheFolder = new File(baseDir + File.separator + ".vripper" + File.separator + "cache");
+        cacheFolder.mkdirs();
+        if (!cacheFolder.exists()) {
+            throw new Exception(String.format("%s could not be created", cacheFolder.toString()));
+        }
+        thumbnails = CacheBuilder.newBuilder()
+                .maximumSize(20000)
+                .build(loader);
+    }
+
+    public void clearCache() {
+        thumbnails.invalidateAll();
+        for (File file : Optional.ofNullable(cacheFolder.listFiles()).orElse(new File[]{})) {
+            FileSystemUtils.deleteRecursively(file);
+        }
+    }
+
+    public long cacheSize() {
+
+        final AtomicLong size = new AtomicLong(0);
+
+        try {
+            Files.walkFileTree(cacheFolder.toPath(), new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+
+                    System.out.println("skipped: " + file + " (" + exc + ")");
+                    // Skip folders that can't be traversed
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+
+                    if (exc != null)
+                        System.out.println("had trouble traversing: " + dir + " (" + exc + ")");
+                    else
+                        size.addAndGet(dir.toFile().length());
+                    // Ignore errors traversing a folder
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new AssertionError("walkFileTree will not throw IOException if the FileVisitor does not");
+        }
+
+        return size.get();
+    }
+
+    private File generateThumbnail(File inputFile, String postId) throws Exception {
+        if (!inputFile.exists()) {
+            throw new Exception(String.format("Input file %s does not exist", inputFile.toString()));
+        }
+        File postsCacheFolder = new File(cacheFolder, postId);
+        postsCacheFolder.mkdirs();
+        if (!postsCacheFolder.exists()) {
+            throw new Exception(String.format("%s could not be created", postsCacheFolder.toString()));
+        }
+
+        File thumbFile = new File(postsCacheFolder, inputFile.getName());
+        if (thumbFile.exists()) {
+            return thumbFile;
+        }
+
+        BufferedImage image = ImageIO.read(inputFile);
+        BufferedImage resize = Scalr.resize(image, 350, 350);
+        ImageIO.write(resize, "jpg", thumbFile);
+        image.flush();
+        resize.flush();
+        return thumbFile;
+    }
+
+    @Getter
+    @Setter
+    @NoArgsConstructor
+    public static class CacheKey {
+        private String postId;
+        private String imgName;
+
+        public CacheKey(String postId, String imgName) {
+            this.postId = postId;
+            this.imgName = imgName;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            CacheKey cacheKey = (CacheKey) o;
+            return Objects.equals(postId, cacheKey.postId) &&
+                    Objects.equals(imgName, cacheKey.imgName);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(postId, imgName);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VGHandler.java

@@ -0,0 +1,86 @@
+package tn.mnlr.vripper.services;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.VripperApplication;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+@Service
+public class VGHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(VGHandler.class);
+
+    @Autowired
+    private AppStateService appStateService;
+
+    @Autowired
+    private PostParser postParser;
+
+    @Getter
+    private LoadingCache<String, List<VRPostState>> cache;
+
+    @PostConstruct
+    private void init() {
+
+        CacheLoader<String, List<VRPostState>> loader = new CacheLoader<>() {
+            @Override
+            public List<VRPostState> load(String threadId) throws Exception {
+                VRThreadParser vrThreadParser = postParser.createVRThreadParser(threadId);
+                List<VRPostState> posts = new ArrayList<>();
+                vrThreadParser.parse();
+                vrThreadParser.getPostPublishProcessor()
+                        .onBackpressureBuffer()
+                        .blockingSubscribe(posts::add);
+                return posts;
+            }
+        };
+
+        cache = CacheBuilder.newBuilder()
+                .expireAfterWrite(30, TimeUnit.MINUTES)
+                .build(loader);
+    }
+
+    public void handle(List<QueuedVGLink> queuedVGLinks) throws Exception {
+        for (QueuedVGLink queuedVGLink : queuedVGLinks) {
+            if (queuedVGLink.getPostId() != null) {
+                postParser.addPost(queuedVGLink.getPostId(), queuedVGLink.getThreadId());
+            } else {
+                appStateService.getGrabQueue().put(queuedVGLink.getLink(), queuedVGLink);
+                appStateService.getLiveGrabQueue().onNext(queuedVGLink);
+
+                Callable<Void> cl = () -> {
+                    List<VRPostState> vrPostStates = cache.get(queuedVGLink.getThreadId());
+                    logger.debug(String.format("%d found for %s", vrPostStates.size(), queuedVGLink.getLink()));
+                    if (vrPostStates.size() == 1) {
+                        postParser.addPost(vrPostStates.get(0).getPostId(), vrPostStates.get(0).getThreadId());
+                        remove(queuedVGLink.getLink());
+                        logger.debug(String.format("threadId %s, postId %s is added automatically for download", queuedVGLink.getThreadId(), queuedVGLink.getPostId()));
+                    }
+
+                    return null;
+                };
+
+                VripperApplication.commonExecutor.submit(cl);
+            }
+        }
+    }
+
+    public void remove(String url) {
+        Optional.ofNullable(appStateService.getGrabQueue().remove(url)).ifPresent(e -> {
+            e.setRemoved(true);
+            appStateService.getLiveGrabQueue().onNext(e);
+        });
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostParser.java

@@ -0,0 +1,190 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.exception.DownloadException;
+import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.host.Host;
+
+import javax.xml.parsers.SAXParserFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static tn.mnlr.vripper.services.PostParser.VR_API;
+
+class VRPostParser {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRPostParser.class);
+
+    private static SAXParserFactory factory = SAXParserFactory.newInstance();
+
+    private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+            .handleIf(e -> e instanceof IOException)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+            .abortOn(InterruptedException.class)
+            .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+
+    private final String threadId;
+    private final String postId;
+    private final ConnectionManager cm;
+    private final VipergirlsAuthService vipergirlsAuthService;
+    private final List<Host> supportedHosts;
+
+    VRPostParser(String threadId, String postId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> supportedHosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.cm = cm;
+        this.vipergirlsAuthService = vipergirlsAuthService;
+        this.supportedHosts = supportedHosts;
+    }
+
+    public Post parse() throws PostParseException {
+
+        logger.debug(String.format("Parsing post %s", postId));
+        HttpGet httpGet;
+        try {
+            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            uriBuilder.setParameter("p", postId);
+            httpGet = cm.buildHttpGet(uriBuilder.build());
+        } catch (URISyntaxException e) {
+            throw new PostParseException(e);
+        }
+        VRPostHandler handler = new VRPostHandler(threadId, postId, supportedHosts);
+        AtomicReference<Throwable> thr = new AtomicReference<>();
+        logger.debug(String.format("Requesting %s", httpGet));
+        Post post = Failsafe.with(retryPolicy)
+                .onFailure(e -> thr.set(e.getFailure()))
+                .get(() -> {
+                    HttpClient connection = cm.getClient().build();
+                    try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
+                        if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                            throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                        }
+
+                        try {
+                            factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                            return handler.getParsedPost();
+                        } catch (Exception e) {
+                            throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
+                        } finally {
+                            EntityUtils.consumeQuietly(response.getEntity());
+                        }
+                    }
+                });
+        if (thr.get() != null || post == null) {
+            logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
+            throw new PostParseException(thr.get());
+        }
+        return post;
+    }
+}
+
+class VRPostHandler extends DefaultHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRPostHandler.class);
+
+    private final List<Host> supportedHosts;
+    private final String threadId;
+    private final String postId;
+    private List<Image> images = new ArrayList<>();
+    private List<String> previews = new ArrayList<>();
+    private String threadTitle;
+    private String postTitle;
+    private String forum;
+    private int previewCounter = 0;
+    private int index = 0;
+    private int imageCount;
+
+    @Getter
+    private Post parsedPost;
+
+    VRPostHandler(String threadId, String postId, List<Host> supportedHosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.supportedHosts = supportedHosts;
+    }
+
+    @Override
+    public void startDocument() {
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) {
+
+        switch (qName.toLowerCase()) {
+            case "forum":
+                forum = attributes.getValue("title").trim();
+                break;
+            case "thread":
+                threadTitle = attributes.getValue("title").trim();
+                break;
+            case "post":
+                imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
+                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                break;
+            case "image":
+                index++;
+                if (previewCounter++ < 4) {
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
+                }
+
+                String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
+                if (mainUrl != null) {
+                    Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
+                    if (foundHost != null) {
+                        logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
+                        images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
+                    } else {
+                        logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
+                    }
+                }
+                break;
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName) {
+        if ("post".equals(qName.toLowerCase())) {
+            if (imageCount != 0) {
+                HashMap<String, Object> metadata = new HashMap<>();
+                metadata.put("PREVIEWS", previews);
+                parsedPost = new Post(
+                        postTitle,
+                        String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
+                        images,
+                        metadata,
+                        postId,
+                        threadId,
+                        threadTitle,
+                        forum
+                );
+            }
+            index = 0;
+            previewCounter = 0;
+            previews = new ArrayList<>();
+            images = new ArrayList<>();
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostState.java

@@ -0,0 +1,29 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+
+import java.util.List;
+
+@Getter
+public class VRPostState {
+    private final String type = "postParse";
+    private final String threadId;
+    private String postId;
+    private int number;
+    private String title;
+    private int imageCount;
+    private String url;
+    private List<String> previews;
+    private String hosts;
+
+    VRPostState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.number = number;
+        this.title = title;
+        this.imageCount = imageCount;
+        this.previews = previews;
+        this.url = url;
+        this.hosts = hosts;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRThreadParser.java

@@ -0,0 +1,176 @@
+package tn.mnlr.vripper.services;
+
+import io.reactivex.processors.ReplayProcessor;
+import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import tn.mnlr.vripper.exception.DownloadException;
+import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.host.Host;
+
+import javax.xml.parsers.SAXParserFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static tn.mnlr.vripper.services.PostParser.VR_API;
+
+public class VRThreadParser {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRThreadParser.class);
+
+    private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+            .handleIf(e -> e instanceof IOException)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+            .abortOn(InterruptedException.class)
+            .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+    private static SAXParserFactory factory = SAXParserFactory.newInstance();
+    @Getter
+    private final ReplayProcessor<VRPostState> postPublishProcessor = ReplayProcessor.create();
+    private final String threadId;
+    private final ConnectionManager cm;
+    private final VipergirlsAuthService vipergirlsAuthService;
+    private final List<Host> hosts;
+
+
+    VRThreadParser(String threadId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> hosts) {
+        this.threadId = threadId;
+        this.cm = cm;
+        this.vipergirlsAuthService = vipergirlsAuthService;
+        this.hosts = hosts;
+    }
+
+    public Void parse() throws PostParseException {
+
+        logger.debug(String.format("Parsing thread %s", threadId));
+        HttpGet httpGet;
+        try {
+            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            uriBuilder.setParameter("t", threadId);
+            httpGet = cm.buildHttpGet(uriBuilder.build());
+        } catch (URISyntaxException e) {
+            throw new PostParseException(e);
+        }
+
+        VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor, hosts);
+        AtomicReference<Throwable> thr = new AtomicReference<>();
+        logger.debug(String.format("Requesting %s", httpGet));
+        Failsafe.with(retryPolicy)
+                .onFailure(e -> thr.set(e.getFailure()))
+                .get(() -> {
+                    HttpClient connection = cm.getClient().build();
+                    try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
+                        if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                            throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                        }
+
+                        try {
+                            factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                        } catch (Exception e) {
+                            throw new PostParseException(String.format("Failed to parse thread %s", threadId), e);
+                        } finally {
+                            EntityUtils.consumeQuietly(response.getEntity());
+                        }
+                    }
+                    return null;
+                });
+        if (thr.get() != null) {
+            logger.error(String.format("parsing failed for thread %s", threadId), thr.get());
+            throw new PostParseException(thr.get());
+        }
+        return null;
+    }
+}
+
+class VRThreadHandler extends DefaultHandler {
+
+    private final String threadId;
+    private final ReplayProcessor<VRPostState> vrPostPublishProcessor;
+    private final List<Host> supportedHosts;
+    private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
+    private List<String> previews = new ArrayList<>();
+    private String threadTitle;
+    private String postId;
+    private String postTitle;
+    private int imageCount;
+    private int postCounter;
+    private int previewCounter = 0;
+
+    VRThreadHandler(String threadId, ReplayProcessor<VRPostState> vrPostPublishProcessor, List<Host> supportedHosts) {
+        this.vrPostPublishProcessor = vrPostPublishProcessor;
+        this.threadId = threadId;
+        this.supportedHosts = supportedHosts;
+    }
+
+    @Override
+    public void startDocument() {
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) {
+
+        switch (qName.toLowerCase()) {
+            case "thread":
+                threadTitle = attributes.getValue("title").trim();
+                break;
+            case "post":
+                imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
+                postId = attributes.getValue("id").trim();
+                postCounter = Integer.parseInt(attributes.getValue("number").trim());
+                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                break;
+            case "image":
+                Optional.ofNullable(attributes.getValue("main_url"))
+                        .map(String::trim)
+                        .flatMap(mainUrl -> supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
+                        .ifPresent(host -> Optional.ofNullable(hostMap.get(host)).ifPresentOrElse(AtomicInteger::incrementAndGet, () -> hostMap.put(host, new AtomicInteger(0))));
+                if (previewCounter++ < 4) {
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
+                }
+                break;
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName) {
+        if ("post".equals(qName.toLowerCase())) {
+            if (imageCount != 0) {
+                vrPostPublishProcessor.onNext(new VRPostState(
+                        threadId,
+                        postId,
+                        postCounter,
+                        postTitle,
+                        imageCount,
+                        String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
+                        previews,
+                        hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
+                ));
+            }
+            previewCounter = 0;
+            previews = new ArrayList<>();
+            hostMap.clear();
+        }
+    }
+
+    @Override
+    public void endDocument() {
+        vrPostPublishProcessor.onComplete();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java

@@ -7,6 +7,9 @@ import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
@@ -18,6 +21,7 @@ import org.w3c.dom.Document;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.VripperException;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -42,7 +46,7 @@ public class VipergirlsAuthService {
     private AppSettingsService appSettingsService;
 
     @Getter
-    private String cookies;
+    private HttpClientContext context = HttpClientContext.create();
 
     private boolean authenticated = false;
 
@@ -52,6 +56,16 @@ public class VipergirlsAuthService {
     @Getter
     private PublishProcessor<String> loggedInUser = PublishProcessor.create();
 
+    @PostConstruct
+    private void init() {
+        context.setCookieStore(new BasicCookieStore());
+        try {
+            authenticate();
+        } catch (VripperException e) {
+            logger.error("Cannot authenticate user with ViperGirls", e);
+        }
+    }
+
     @PreDestroy
     private void destroy() throws InterruptedException {
         logger.info("Shutting down VipergirlsAuthService");
@@ -65,8 +79,8 @@ public class VipergirlsAuthService {
         authenticated = false;
 
         if(!appSettingsService.isVLogin()) {
-            logger.warn("Authentication option is disabled");
-            cookies = null;
+            logger.debug("Authentication option is disabled");
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             return;
@@ -77,7 +91,7 @@ public class VipergirlsAuthService {
 
         if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
             logger.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
-            cookies = null;
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             return;
@@ -97,7 +111,7 @@ public class VipergirlsAuthService {
         try {
             postAuth.setEntity(new UrlEncodedFormEntity(params));
         } catch (Exception e) {
-            cookies = null;
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             throw new VripperException(e);
@@ -108,28 +122,19 @@ public class VipergirlsAuthService {
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(postAuth)) {
+        try (CloseableHttpResponse response = client.execute(postAuth, context)) {
 
             if (response.getStatusLine().getStatusCode() / 100 != 2) {
                 throw new VripperException(String.format("Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
             }
-            StringBuilder sb = new StringBuilder();
-            Arrays.asList(response.getHeaders("set-cookie"))
-                    .stream()
-                    .map(e -> e.getValue())
-                    .map(e -> e.split(";")[0])
-                    .filter(e -> e.startsWith("vg_lastactivity") || e.startsWith("vg_userid") || e.startsWith("vg_password"))
-                    .forEach(e -> sb.append(e).append(";"));
-            String cookies = sb.toString();
             String responseBody = EntityUtils.toString(response.getEntity());
             logger.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
             EntityUtils.consumeQuietly(response.getEntity());
-            if(!cookies.contains("vg_userid")) {
+            if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) {
                 throw new VripperException("Failed to authenticate user with ViperRipper");
             }
-            this.cookies = cookies;
         } catch (Exception e) {
-            cookies = null;
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             if (e instanceof VripperException) {
@@ -144,14 +149,14 @@ public class VipergirlsAuthService {
         loggedInUser.onNext(loggedUser);
     }
 
-    public void leaveThanks(String postUrl, String postId) {
+    void leaveThanks(String postUrl, String postId) {
         VripperApplication.commonExecutor.submit(() -> {
             if (!appSettingsService.isVLogin()) {
-                logger.warn("Authentication with ViperGirls option is disabled");
+                logger.debug("Authentication with ViperGirls option is disabled");
                 return;
             }
             if (!appSettingsService.isVThanks()) {
-                logger.warn("Leave thanks option is disabled");
+                logger.debug("Leave thanks option is disabled");
                 return;
             }
             if (!authenticated) {
@@ -168,16 +173,15 @@ public class VipergirlsAuthService {
 
     private String getSecurityToken(String url) throws VripperException {
 
-        String securityToken = "";
+        String securityToken;
 
         HttpGet httpGet = cm.buildHttpGet(url);
         httpGet.addHeader("Referer", "https://vipergirls.to/");
         httpGet.addHeader("Host", "vipergirls.to");
-        httpGet.addHeader("Cookie", cookies);
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(httpGet)) {
+        try (CloseableHttpResponse response = client.execute(httpGet, context)) {
 
             String postPage = EntityUtils.toString(response.getEntity());
             Document document = htmlProcessorService.clean(postPage);
@@ -189,7 +193,7 @@ public class VipergirlsAuthService {
                     .getTextContent()
                     .trim();
 
-            securityToken = Arrays.asList(thanksUrl.split("&amp;")).stream()
+            securityToken = Arrays.stream(thanksUrl.split("&amp;"))
                     .filter(v -> v.startsWith("securitytoken"))
                     .findAny()
                     .orElse("")
@@ -218,11 +222,10 @@ public class VipergirlsAuthService {
 
         postThanks.addHeader("Referer", "https://vipergirls.to/");
         postThanks.addHeader("Host", "vipergirls.to");
-        postThanks.addHeader("Cookie", cookies);
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(postThanks)) {
+        try (CloseableHttpResponse response = client.execute(postThanks, context)) {
             EntityUtils.consumeQuietly(response.getEntity());
         } catch (Exception e) {
             throw new VripperException(e);

vripper-server/src/main/java/tn/mnlr/vripper/services/XpathService.java

@@ -2,7 +2,6 @@ package tn.mnlr.vripper.services;
 
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 import tn.mnlr.vripper.exception.XpathException;
 
 import javax.xml.xpath.XPath;
@@ -12,20 +11,8 @@ import javax.xml.xpath.XPathFactory;
 @Service
 public class XpathService {
 
-    private XpathService() {
-
-    }
-
     private XPath xpath = XPathFactory.newInstance().newXPath();
 
-    public NodeList getAsNodeList(Node source, String xpathExpression) throws XpathException {
-        try {
-            return (NodeList) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET);
-        } catch (Exception e) {
-            throw new XpathException(e);
-        }
-    }
-
     public Node getAsNode(Node source, String xpathExpression) throws XpathException {
         try {
             return (Node) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE);

vripper-server/src/main/java/tn/mnlr/vripper/web/WebSocketConfig.java

@@ -16,6 +16,6 @@ public class WebSocketConfig implements WebSocketConfigurer {
 
     @Override
     public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
-        registry.addHandler(handler, "/endpoint").setAllowedOrigins("*").withSockJS();
+        registry.addHandler(handler, "/endpoint").setAllowedOrigins("*").withSockJS().setClientLibraryUrl("../../assets/sockjs.min.js");
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/GalleryEndpoint.java

@@ -0,0 +1,202 @@
+package tn.mnlr.vripper.web.restendpoints;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.CacheControl;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import tn.mnlr.vripper.services.AppSettingsService;
+import tn.mnlr.vripper.services.PathService;
+import tn.mnlr.vripper.services.ThumbnailGenerator;
+
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+import javax.servlet.http.HttpServletResponse;
+import java.awt.*;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+@RestController
+@CrossOrigin(value = "*")
+public class GalleryEndpoint {
+
+    private static final Logger logger = LoggerFactory.getLogger(GalleryEndpoint.class);
+
+    @Autowired
+    private PathService pathService;
+
+    @Autowired
+    private ThumbnailGenerator thumbnailGenerator;
+
+    private static final String cacheControl = CacheControl.maxAge(365, TimeUnit.DAYS).getHeaderValue();
+    @Autowired
+    private AppSettingsService appSettingsService;
+
+    @ExceptionHandler(Exception.class)
+    public ResponseEntity handleException(Exception e) {
+        logger.error("Error when process request", e);
+        return ResponseEntity
+                .status(HttpStatus.INTERNAL_SERVER_ERROR)
+                .body(e.getMessage());
+    }
+
+    @GetMapping(value = "/image/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<byte[]> getImage(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        response.addHeader("Cache-Control", cacheControl);
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        return ResponseEntity.ok(Files.readAllBytes(Paths.get(destinationFolder.toPath().toString(), imgName)));
+    }
+
+    @GetMapping(value = "/image/thumb/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<byte[]> getImageThumb(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        response.addHeader("Cache-Control", cacheControl);
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        if (destinationFolder.exists() && destinationFolder.isDirectory()) {
+            return ResponseEntity.ok(thumbnailGenerator.getThumbnails().get(new ThumbnailGenerator.CacheKey(postId, imgName)));
+        }
+        return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
+    }
+
+    @GetMapping("/gallery/{postId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<List<GalleryImage>> getGallery(@PathVariable("postId") @NonNull String postId) {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        if (destinationFolder.exists() && destinationFolder.isDirectory()) {
+            return ResponseEntity.ok(
+                    Arrays.stream(Objects.requireNonNull(destinationFolder.listFiles()))
+                            .filter(f -> !f.getName().endsWith("tmp"))
+                            .filter(f -> f.getName().toLowerCase().endsWith(".jpg") || f.getName().toLowerCase().endsWith(".jpeg"))
+                            .sorted(Comparator.comparing(File::getName))
+                            .map(GalleryImage::fromFile)
+                            .filter(Objects::nonNull)
+                            .collect(Collectors.toList())
+            );
+        }
+        return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
+    }
+
+    @GetMapping("/gallery/cache")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> getCacheSize() {
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    @GetMapping("/gallery/cache/clear")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> clearCache() {
+        thumbnailGenerator.clearCache();
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    private String humanReadableByteCount(long bytes, boolean si) {
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class CacheSize {
+
+    private String size;
+
+    public CacheSize(String size) {
+        this.size = size;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class GalleryImage {
+
+    private static final CacheLoader<File, Dimension> loader = new CacheLoader<>() {
+
+        @Override
+        public Dimension load(File file) {
+            try (ImageInputStream in = ImageIO.createImageInputStream(file)) {
+                final Iterator<ImageReader> readers = ImageIO.getImageReaders(in);
+                if (readers.hasNext()) {
+                    ImageReader reader = readers.next();
+                    try {
+                        reader.setInput(in);
+                        return new Dimension(reader.getWidth(0), reader.getHeight(0));
+                    } finally {
+                        reader.dispose();
+                    }
+                } else {
+                    logger.error(String.format("No reader found for image %s", file.toString()));
+                    return null;
+                }
+            } catch (Exception e) {
+                logger.error(String.format("Failed to create image object for %s", file.toString()), e);
+                return null;
+            }
+        }
+    };
+
+    private static final LoadingCache<File, Dimension> cache = CacheBuilder.newBuilder()
+            .maximumSize(20000)
+            .build(loader);
+
+    private static final Logger logger = LoggerFactory.getLogger(GalleryImage.class);
+    private String title;
+    private String src;
+    private String msrc;
+    private double w;
+    private double h;
+
+    public GalleryImage(String title, String src, String msrc, double w, double h) {
+        this.title = title;
+        this.src = src;
+        this.msrc = msrc;
+        this.w = w;
+        this.h = h;
+    }
+
+    public static GalleryImage fromFile(File file) {
+
+        Dimension dimension = null;
+        try {
+            dimension = cache.get(file);
+        } catch (ExecutionException e) {
+            logger.error(String.format("Failed to get image dimensions for %s", file.toString()), e);
+        }
+        if (dimension == null) {
+            return null;
+        }
+        return new GalleryImage(file.getName(), file.getName(), file.getName(), dimension.getWidth(), dimension.getHeight());
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java

@@ -1,7 +1,6 @@
 package tn.mnlr.vripper.web.restendpoints;
 
-import lombok.Getter;
-import lombok.ToString;
+import lombok.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -10,16 +9,14 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.PostParseException;
-import tn.mnlr.vripper.q.DownloadQ;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.AppStateService;
-import tn.mnlr.vripper.services.PathService;
-import tn.mnlr.vripper.services.PostParser;
+import tn.mnlr.vripper.q.ExecutionService;
+import tn.mnlr.vripper.services.*;
 
 import java.io.File;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 @RestController
 @CrossOrigin(value = "*")
@@ -27,8 +24,7 @@ public class PostRestEndpoint {
 
     private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
 
-
-    private static final Pattern VG_URL_PATTERN = Pattern.compile("https:\\/\\/vipergirls\\.to\\/threads\\/(\\d+)((.*p=)(\\d+))?");
+    private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?");
 
     @Autowired
     private AppStateService appStateService;
@@ -39,8 +35,11 @@ public class PostRestEndpoint {
     @Autowired
     private PathService pathService;
 
+    @Autowired
+    private VGHandler vgHandler;
+
     @ExceptionHandler(Exception.class)
-    public ResponseEntity handleException(Exception e) {
+    public ResponseEntity<String> handleException(Exception e) {
         logger.error("Error when process request", e);
         return ResponseEntity
                 .status(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -48,51 +47,61 @@ public class PostRestEndpoint {
     }
 
     @Autowired
-    private DownloadQ downloadQ;
+    private ExecutionService executionService;
+//    private DownloadQ downloadQ;
+
 
     @Autowired
     private PostParser postParser;
 
     @PostMapping("/post")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception {
-        logger.info(String.format("Starting to process thread: %s", url.url));
-        if (url.url == null || url.url.isEmpty()) {
-            return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST);
-        } else if (!url.url.startsWith("https://vipergirls.to")) {
-            return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
+    public synchronized ResponseEntity processPost(@RequestBody ThreadUrl _url) throws Exception {
+        if (_url.getUrl() == null || _url.getUrl().isEmpty()) {
+            return new ResponseEntity("Failed to process empty request", HttpStatus.BAD_REQUEST);
         }
-
-        String threadId, postId;
-        try {
-            Matcher m = VG_URL_PATTERN.matcher(url.url);
-            if (m.find()) {
-                threadId = m.group(1);
-                postId = m.group(4);
-            } else {
-                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+        List<String> urls = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
+        for (String url : urls) {
+            logger.debug(String.format("Starting to process thread: %s", url));
+            if (!url.startsWith("https://vipergirls.to")) {
+                logger.error(String.format("Unsupported link %s", url));
+                continue;
             }
-        } catch (Exception e) {
-            throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+
+            String threadId, postId;
+            try {
+                Matcher m = VG_URL_PATTERN.matcher(url);
+                if (m.find()) {
+                    threadId = m.group(1);
+                    postId = m.group(4);
+                } else {
+                    throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+                }
+            } catch (Exception e) {
+                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+            }
+            vgHandler.handle(Collections.singletonList(new QueuedVGLink(url, threadId, postId)));
         }
-        return ResponseEntity.ok(new PairThreadIdPostId(threadId, postId));
+        return ResponseEntity.ok().build();
     }
 
     @PostMapping("/post/restart")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost(@RequestBody PostId postId) throws Exception {
-        downloadQ.restart(postId.getPostId());
+    public synchronized void restartPost(@RequestBody @NonNull List<PostId> postIds) throws Exception {
+        for (PostId postId : postIds) {
+            executionService.restart(postId.getPostId());
+        }
     }
 
     @PostMapping("/post/add")
     @ResponseStatus(value = HttpStatus.OK)
-    public void addPost(@RequestBody List<PostToAdd> posts) {
+    public synchronized void addPost(@RequestBody List<PostToAdd> posts) {
         for (PostToAdd post : posts) {
             VripperApplication.commonExecutor.submit(() -> {
                 try {
-                    postParser.addPost(post.postId, post.threadId);
+                    postParser.addPost(post.getPostId(), post.getThreadId());
                 } catch (PostParseException e) {
-                    logger.error(String.format("Failed to add post %s", post.postId), e);
+                    logger.error(String.format("Failed to add post %s", post.getPostId()), e);
                 }
             });
         }
@@ -107,128 +116,166 @@ public class PostRestEndpoint {
 
     @PostMapping("/post/restart/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost() throws Exception {
-        downloadQ.restartAll();
+    public synchronized void restartPost() throws Exception {
+        executionService.restartAll();
     }
 
     @PostMapping("/post/stop")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stop(@RequestBody PostId postId) {
-        downloadQ.stop(postId.getPostId());
+    public synchronized void stop(@RequestBody @NonNull List<PostId> postIds) {
+        for (PostId postId : postIds) {
+            executionService.stop(postId.getPostId());
+        }
     }
 
     @PostMapping("/post/stop/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stopAll() {
-        downloadQ.stopAll();
+    public synchronized void stopAll() {
+        executionService.stopAll();
     }
 
     @PostMapping("/post/remove")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity remove(@RequestBody PostId postId) {
-        downloadQ.stop(postId.getPostId());
-        appStateService.remove(postId.getPostId());
-        return ResponseEntity.ok(new RemoveResult(postId.getPostId()));
+    public synchronized ResponseEntity<List<RemoveResult>> remove(@RequestBody @NonNull List<PostId> postIds) {
+        List<RemoveResult> result = new ArrayList<>();
+        for (PostId postId : postIds) {
+            executionService.stop(postId.getPostId());
+            appStateService.remove(postId.getPostId());
+            result.add(new RemoveResult(postId.getPostId()));
+        }
+        return ResponseEntity.ok(result);
     }
 
     @PostMapping("/post/clear/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity clearAll() {
+    public synchronized ResponseEntity<RemoveAllResult> clearAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.clearAll()));
     }
 
     @PostMapping("/post/remove/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity removeAll() {
+    public synchronized ResponseEntity<RemoveAllResult> removeAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.removeAll()));
     }
 
-    @Getter
-    @ToString
-    private static class ThreadUrl {
-        private String url;
+    @GetMapping("/grab/{threadId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public synchronized ResponseEntity<List<VRPostState>> grab(@PathVariable("threadId") @NonNull ThreadId threadId) throws Exception {
+        return ResponseEntity.ok(vgHandler.getCache().get(threadId.getThreadId()));
     }
 
-    @Getter
-    private static class PostId {
-        private String postId;
-    }
-
-    @Getter
-    private static class PairThreadIdPostId {
-        private String threadId;
-        private String postId;
-
-        public PairThreadIdPostId(String threadId, String postId) {
-            this.threadId = threadId;
-            this.postId = postId;
-        }
-    }
-
-    @Getter
-    private static class PostToAdd {
-        private String threadId;
-        private String postId;
-    }
-
-    @Getter
-    private static class ParseResult {
-
-        private List<PostResult> posts;
-        private int count;
-        private String threadId;
-
-        public ParseResult(List<PostResult> posts, int count, String threadId) {
-            this.posts = posts;
-            this.count = count;
-            this.threadId = threadId;
-        }
-
-        @Getter
-        public static class PostResult {
-
-            private String title;
-            private int counter;
-            private String url;
-            private String postId;
-            private List<String> previews;
-
-            public PostResult(String title, int counter, String url, String postId, List<String> previews) {
-                this.title = title;
-                this.counter = counter;
-                this.url = url;
-                this.postId = postId;
-                this.previews = previews;
-            }
-        }
-    }
-
-    @Getter
-    private static class RemoveAllResult {
-        private List<String> postIds;
-        private int removed;
-
-        RemoveAllResult(List<String> postIds) {
-            this.removed = postIds.size();
-            this.postIds = postIds;
-        }
-    }
-
-    @Getter
-    private static class RemoveResult {
-        private String postId;
-
-        RemoveResult(String postId) {
-            this.postId = postId;
-        }
-    }
-
-    @Getter
-    private static class DownloadPath {
-        private String path;
-
-        DownloadPath(String path) {
-            this.path = path;
-        }
+    @PostMapping("/grab/remove")
+    @ResponseStatus(value = HttpStatus.OK)
+    public synchronized void grabRemove(@RequestBody @NonNull ThreadUrl threadUrl) {
+        vgHandler.remove(threadUrl.getUrl());
     }
 }
+
+@Getter
+@Setter
+@NoArgsConstructor
+class ThreadUrl {
+    private String url;
+
+    public ThreadUrl(String url) {
+        this.url = url;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class ThreadId {
+    private String threadId;
+
+    public ThreadId(String threadId) {
+        this.threadId = threadId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PostId {
+    private String postId;
+
+    public PostId(String postId) {
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PostToAdd {
+    private String threadId;
+    private String postId;
+
+    public PostToAdd(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class RemoveAllResult {
+    private List<String> postIds;
+    private int removed;
+
+    RemoveAllResult(List<String> postIds) {
+        this.removed = postIds.size();
+        this.postIds = postIds;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class RemoveResult {
+    private String postId;
+
+    RemoveResult(String postId) {
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class DownloadPath {
+    private String path;
+
+    DownloadPath(String path) {
+        this.path = path;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+@ToString
+class PairThreadIdPostId {
+    private String threadId;
+    private String postId;
+
+    public PairThreadIdPostId(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        PairThreadIdPostId that = (PairThreadIdPostId) o;
+        return Objects.equals(threadId, that.threadId) &&
+                Objects.equals(postId, that.postId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(threadId, postId);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -54,10 +54,13 @@ public class SettingsRestEndpoint {
             try {
                 this.settings.check(settings);
             } catch (ValidationException e) {
-                return new ResponseEntity(new Response(e.getMessage()), HttpStatus.BAD_REQUEST);
+                return ResponseEntity
+                        .status(HttpStatus.BAD_REQUEST)
+                        .body(e.getMessage());
             }
             this.settings.setDownloadPath(settings.getDownloadPath());
             this.settings.setMaxThreads(settings.getMaxThreads());
+            this.settings.setMaxTotalThreads(settings.getMaxTotalThreads());
             this.settings.setAutoStart(settings.isAutoStart());
             this.settings.setVLogin(settings.isVLogin());
 
@@ -77,6 +80,7 @@ public class SettingsRestEndpoint {
             this.settings.setSubLocation(settings.isSubLocation());
             this.settings.setThreadSubLocation(settings.isThreadSubLocation());
             this.settings.setClearCompleted(settings.isClearCompleted());
+            this.settings.setViewPhotos(settings.isViewPhotos());
 
             this.settings.save();
 
@@ -92,6 +96,7 @@ public class SettingsRestEndpoint {
         return new AppSettingsService.Settings(
                 settings.getDownloadPath(),
                 settings.getMaxThreads(),
+                settings.getMaxTotalThreads(),
                 settings.isAutoStart(),
                 settings.isVLogin(),
                 settings.getVUsername(),
@@ -101,15 +106,16 @@ public class SettingsRestEndpoint {
                 settings.isForceOrder(),
                 settings.isSubLocation(),
                 settings.isThreadSubLocation(),
-                settings.isClearCompleted()
+                settings.isClearCompleted(),
+                settings.isViewPhotos()
         );
     }
 
     @Getter
-    class Response {
+    static class Response {
         String message;
 
-        public Response(String message) {
+        Response(String message) {
             this.message = message;
         }
     }

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java

@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.schedulers.Schedulers;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,7 +14,6 @@ import org.springframework.web.socket.CloseStatus;
 import org.springframework.web.socket.TextMessage;
 import org.springframework.web.socket.WebSocketSession;
 import org.springframework.web.socket.handler.TextWebSocketHandler;
-import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.ui.ImageUIMixin;
@@ -20,7 +21,7 @@ import tn.mnlr.vripper.entities.mixin.ui.PostUIMixin;
 import tn.mnlr.vripper.services.*;
 
 import java.io.IOException;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
@@ -54,10 +55,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private Map<String, Disposable> postsSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> postDetailsSubscriptions = new ConcurrentHashMap<>();
-    private Map<String, Disposable> vrPostParserSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> stateSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> downloadSpeedSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> userSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> grabQueueSubscriptions = new ConcurrentHashMap<>();
 
     private Map<String, Future<Void>> threadParseRequests = new ConcurrentHashMap<>();
 
@@ -69,6 +70,9 @@ public class WebSocketHandler extends TextWebSocketHandler {
         WSMessage wsMessage = om.readValue(message.getPayload(), WSMessage.class);
         WSMessage.CMD cmd = WSMessage.CMD.valueOf(wsMessage.getCmd());
         switch (cmd) {
+            case GRAB_QUEUE_SUB:
+                subscribeForGrabQueue(session);
+                break;
             case GLOBAL_STATE_SUB:
                 subscribeForGlobalState(session);
                 break;
@@ -81,49 +85,45 @@ public class WebSocketHandler extends TextWebSocketHandler {
             case POSTS_SUB:
                 subscribeForPosts(session);
                 break;
-            case THREAD_PARSING_SUB:
-                subscribeForThreadParsing(session, wsMessage.getPayload());
-                break;
             case POST_DETAILS_SUB:
                 subscribeForPostDetails(session, wsMessage.getPayload());
                 break;
             case POST_DETAILS_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from post details", session.getId()));
-                Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-                break;
-            case THREAD_PARSING_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from thread parsing", session.getId()));
-                Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-                Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
+                logger.debug(String.format("Client %s unsubscribed from post details", session.getId()));
+                Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case POSTS_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from posts", session.getId()));
-                Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from posts", session.getId()));
+                Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case GLOBAL_STATE_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from global state", session.getId()));
-                Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from global state", session.getId()));
+                Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case SPEED_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from download speed info", session.getId()));
-                Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from download speed info", session.getId()));
+                Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case USER_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from user info", session.getId()));
-                Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from user info", session.getId()));
+                Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
+            case GRAB_QUEUE_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from grab queue", session.getId()));
+                Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
         }
     }
 
     private void subscribeForGlobalState(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for global state", session.getId()));
+        logger.debug(String.format("Client %s subscribed for global state", session.getId()));
         if (stateSubscriptions.containsKey(session.getId())) {
             stateSubscriptions.get(session.getId()).dispose();
         }
 
         try {
-            send(session, new TextMessage(om.writeValueAsString(Arrays.asList(globalStateService.getCurrentState()))));
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(globalStateService.getCurrentState()))));
         } catch (Exception e) {
             logger.error("Unexpected error occurred", e);
         }
@@ -143,13 +143,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private void subscribeForSpeed(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for download speed info", session.getId()));
+        logger.debug(String.format("Client %s subscribed for download speed info", session.getId()));
         if (downloadSpeedSubscriptions.containsKey(session.getId())) {
             downloadSpeedSubscriptions.get(session.getId()).dispose();
         }
 
         try {
-            send(session, new TextMessage(om.writeValueAsString(Arrays.asList(new DownloadSpeed(0)))));
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new DownloadSpeed(0)))));
         } catch (Exception e) {
             logger.error("Unexpected error occurred", e);
         }
@@ -170,13 +170,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private void subscribeForUser(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for user info", session.getId()));
+        logger.debug(String.format("Client %s subscribed for user info", session.getId()));
         if (userSubscriptions.containsKey(session.getId())) {
             userSubscriptions.get(session.getId()).dispose();
         }
 
         try {
-            send(session, new TextMessage(om.writeValueAsString(Arrays.asList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
         } catch (Exception e) {
             logger.error("Unexpected error occurred", e);
         }
@@ -185,7 +185,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 vipergirlsAuthService.getLoggedInUser()
                         .onBackpressureBuffer()
                         .observeOn(Schedulers.io())
-                        .map(e -> Arrays.asList(new LoggedUser(e)))
+                        .map(e -> Collections.singletonList(new LoggedUser(e)))
                         .map(om::writeValueAsString)
                         .map(TextMessage::new)
                         .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
@@ -194,7 +194,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private void subscribeForPosts(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for posts", session.getId()));
+        logger.debug(String.format("Client %s subscribed for posts", session.getId()));
         if (postsSubscriptions.containsKey(session.getId())) {
             postsSubscriptions.get(session.getId()).dispose();
         }
@@ -218,35 +218,35 @@ public class WebSocketHandler extends TextWebSocketHandler {
         );
     }
 
-    private void subscribeForThreadParsing(WebSocketSession session, String threadId) {
+    private void subscribeForGrabQueue(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for thread parsing with threadId = %s", session.getId(), threadId));
-        if (vrPostParserSubscriptions.containsKey(session.getId())) {
-            vrPostParserSubscriptions.get(session.getId()).dispose();
+        logger.debug(String.format("Client %s subscribed for grab queue", session.getId()));
+        if (grabQueueSubscriptions.containsKey(session.getId())) {
+            grabQueueSubscriptions.get(session.getId()).dispose();
         }
 
-        if (threadParseRequests.containsKey(session.getId())) {
-            threadParseRequests.get(session.getId()).cancel(true);
+        try {
+            send(session, new TextMessage(om.writeValueAsString(appStateService.getGrabQueue().values())));
+        } catch (Exception e) {
+            logger.error("Unexpected error occurred", e);
         }
 
-        PostParser.VRThreadParser vrThreadParser = postParser.new VRThreadParser(threadId);
-
-        vrPostParserSubscriptions.put(session.getId(), vrThreadParser.getPostPublishProcessor()
-                .onBackpressureBuffer()
-                .observeOn(Schedulers.io())
-                .buffer(200)
-                .filter(e -> !e.isEmpty())
-                .map(om::writeValueAsString)
-                .map(TextMessage::new)
-                .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        grabQueueSubscriptions.put(session.getId(),
+                appStateService.getLiveGrabQueue()
+                        .onBackpressureBuffer()
+                        .observeOn(Schedulers.io())
+                        .buffer(2000, TimeUnit.MILLISECONDS, 200)
+                        .filter(e -> !e.isEmpty())
+                        .map(e -> e.stream().distinct().collect(Collectors.toList()))
+                        .map(om::writeValueAsString)
+                        .map(TextMessage::new)
+                        .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
         );
-
-        threadParseRequests.put(session.getId(), VripperApplication.commonExecutor.submit(vrThreadParser::parse));
     }
 
     private void subscribeForPostDetails(WebSocketSession session, String postId) {
 
-        logger.info(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
+        logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
         if (postDetailsSubscriptions.containsKey(session.getId())) {
             postDetailsSubscriptions.get(session.getId()).dispose();
         }
@@ -282,24 +282,26 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     @Override
     public void afterConnectionEstablished(WebSocketSession session) {
-        logger.info(String.format("Connection open for client id: %s", session.getId()));
+        logger.debug(String.format("Connection open for client id: %s", session.getId()));
     }
 
     @Override
     public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
 
-        Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+        Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
         Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
 
-        logger.info(String.format("Connection closed for client id: %s", session.getId()));
+        logger.debug(String.format("Connection closed for client id: %s", session.getId()));
     }
 
     @Getter
+    @Setter
+    @NoArgsConstructor
     private static class WSMessage {
 
         private String cmd;
@@ -314,10 +316,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
             GLOBAL_STATE_UNSUB,
             SPEED_SUB,
             SPEED_UNSUB,
-            THREAD_PARSING_SUB,
-            THREAD_PARSING_UNSUB,
             USER_SUB,
-            USER_UNSUB
+            USER_UNSUB,
+            GRAB_QUEUE_SUB,
+            GRAB_QUEUE_UNSUB
         }
     }
 
@@ -327,7 +329,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
         private final String type = "user";
         private String user;
 
-        public LoggedUser(String user) {
+        LoggedUser(String user) {
             this.user = user;
         }
     }

vripper-ui/angular.json

@@ -27,9 +27,14 @@
               "src/assets"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "./node_modules/photoswipe/dist/photoswipe.css",
+              "./node_modules/photoswipe/dist/default-skin/default-skin.css"
             ],
-            "scripts": []
+            "scripts": [
+              "./node_modules/photoswipe/dist/photoswipe.min.js",
+              "./node_modules/photoswipe/dist/photoswipe-ui-default.min.js"
+            ]
           },
           "configurations": {
             "production": {
@@ -83,9 +88,14 @@
             "tsConfig": "src/tsconfig.spec.json",
             "karmaConfig": "src/karma.conf.js",
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "./node_modules/photoswipe/dist/photoswipe.css",
+              "./node_modules/photoswipe/dist/default-skin/default-skin.css"
+            ],
+            "scripts": [
+              "./node_modules/photoswipe/dist/photoswipe.min.js",
+              "./node_modules/photoswipe/dist/photoswipe-ui-default.min.js"
             ],
-            "scripts": [],
             "assets": [
               "src/favicon.ico",
               "src/assets"

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.2.3",
+  "version": "2.10.3",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -13,36 +13,37 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^7.2.3",
-    "@angular/cdk": "^7.3.1",
-    "@angular/common": "~7.2.0",
-    "@angular/compiler": "~7.2.0",
-    "@angular/core": "~7.2.0",
-    "@angular/flex-layout": "^7.0.0-beta.23",
-    "@angular/forms": "~7.2.0",
-    "@angular/material": "^7.3.0",
-    "@angular/platform-browser": "~7.2.0",
-    "@angular/platform-browser-dynamic": "~7.2.0",
-    "@angular/router": "~7.2.0",
-    "ag-grid-angular": "^21.2.1",
-    "ag-grid-community": "^21.2.1",
+    "@angular/animations": "^8.2.14",
+    "@angular/cdk": "^8.2.3",
+    "@angular/common": "~8.2.14",
+    "@angular/compiler": "~8.2.14",
+    "@angular/core": "~8.2.14",
+    "@angular/flex-layout": "^8.0.0-beta.27",
+    "@angular/forms": "~8.2.14",
+    "@angular/material": "^8.2.3",
+    "@angular/platform-browser": "~8.2.14",
+    "@angular/platform-browser-dynamic": "~8.2.14",
+    "@angular/router": "~8.2.14",
+    "ag-grid-angular": "^22.1.0",
+    "ag-grid-community": "^22.1.0",
     "core-js": "^2.5.4",
     "hammerjs": "^2.0.8",
-    "ngx-electron": "^2.1.1",
-    "rxjs": "~6.3.3",
+    "ngx-electron": "^2.2.0",
+    "photoswipe": "^4.1.3",
+    "rxjs": "~6.5.3",
     "tslib": "^1.9.0",
     "zone.js": "~0.8.26"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^0.13.9",
-    "@angular/cli": "~7.2.2",
-    "@angular/compiler-cli": "^7.2.15",
-    "@angular/language-service": "~7.2.0",
+    "@angular-devkit/build-angular": "^0.803.21",
+    "@angular/cli": "~8.3.20",
+    "@angular/compiler-cli": "^8.2.14",
+    "@angular/language-service": "~8.2.14",
     "@types/jasmine": "~2.8.8",
     "@types/jasminewd2": "~2.0.3",
     "@types/node": "~8.9.4",
     "codelyzer": "~4.5.0",
-    "electron": "^6.0.9",
+    "electron": "^7.1.7",
     "jasmine-core": "~2.99.1",
     "jasmine-spec-reporter": "~4.2.1",
     "karma": "^4.2.0",
@@ -53,6 +54,6 @@
     "protractor": "~5.4.0",
     "ts-node": "~7.0.0",
     "tslint": "~5.11.0",
-    "typescript": "~3.2.2"
+    "typescript": "~3.5.0"
   }
 }

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.2.3</version>
+    <version>2.10.3</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.component.html

@@ -1,45 +1,25 @@
-<div fxLayout="column" *ngIf="connecting()" class="overlay loading" fxLayoutAlign="center center">
-  <mat-spinner></mat-spinner>
-  <h2>Loading...</h2>
-</div>
-<div fxLayout="column" *ngIf="noConnectionState()" class="overlay no-connection" fxLayoutAlign="center center">
-  <mat-icon class="overlay-icon">error</mat-icon>
-  <h2>The app seems to be shutdown</h2>
-</div>
-<div [ngClass]="{'electron': electronService.isElectronApp}" id="app-container" fxLayout="column" fxLayoutGap="20px">
-  <div id="app-header" fxFlex="nogrow">
-    <mat-toolbar class="mat-elevation-z8" color="accent">
-      <mat-toolbar-row fxLayout="row" fxLayoutAlign="center center" fxLayoutGap="10px">
-        <div id="left-controls">
-          <button (click)="scan()" class="add-button" color="primary" mat-icon-button>
-            <mat-icon>add</mat-icon>
-          </button>
-          <button (click)="restartAll()" aria-label="Start" mat-icon-button title="Start">
-            <mat-icon>play_arrow</mat-icon>
-          </button>
-          <button (click)="stopAll()" aria-label="Stop" mat-icon-button title="Stop">
-            <mat-icon>stop</mat-icon>
-          </button>
-          <button (click)="clear()" aria-label="Clear completed" mat-icon-button title="Clear completed">
-            <mat-icon>clear_all</mat-icon>
-          </button>
-          <button (click)="remove()" aria-label="Remove All" mat-icon-button title="Remove All">
-            <mat-icon>delete</mat-icon>
-          </button>
-        </div>
-        <span fxFlex="grow" fxLayoutAlign="end center">
-          <p style="font-size: 16px">Logged in as: {{loggedUser.user == null || loggedUser.user === '' ? 'guest': loggedUser.user }}</p>
-        </span>
-        <div id="right-controls">
-          <button (click)="openSettings()" aria-label="settings" mat-icon-button>
-            <mat-icon>menu</mat-icon>
-          </button>
-        </div>
-      </mat-toolbar-row>
-    </mat-toolbar>
+<ng-container *ngIf="appState | async as state">
+  <div *ngIf="state === 'CONNECTING'" class="overlay loading" fxLayout="column" fxLayoutAlign="center center">
+    <div class="sk-chase">
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+    </div>
   </div>
-  <div id="app-body" fxFlex="grow">
-    <router-outlet></router-outlet>
+  <div *ngIf="state === 'DISCONNECTED'" class="overlay no-connection" fxLayout="column" fxLayoutAlign="center center">
+    <mat-icon class="overlay-icon">error</mat-icon>
+    <h2>Error connecting to daemon, try restarting the application</h2>
+  </div>
+  <div *ngIf="state === 'CONNECTED'" [ngClass]="{ electron: electron | async }" fxLayout="column" id="app-container">
+    <div fxFlex="nogrow" id="app-header">
+      <app-toolbar></app-toolbar>
+    </div>
+    <div fxFlex="grow" id="app-body" style="margin-bottom: 20px;">
+      <router-outlet></router-outlet>
+    </div>
     <app-status-bar style="width: 100%"></app-status-bar>
   </div>
-</div>
+</ng-container>

vripper-ui/src/app/app.component.scss

@@ -1,21 +1,76 @@
-#logo {
-  height: 48px;
-  margin-right: 10px;
-  background-image: url('../assets/logo.png');
-  background-repeat: no-repeat;
-  background-size: 48px 48px;
-}
-
-#title {
-  user-select: none
-}
-
 .overlay-icon {
-  font-size: 128px;
-  width: 128px;
-  height: 128px;
+  font-size: 64px;
+  width: 64px;
+  height: 64px;
 }
 
-.no-connection {
-  color: #eb6060;
+.no-connection, .loading {
+  background-color: #2c3e50;
 }
+
+.overlay {
+  position: absolute;
+  display: block;
+  width: 100%;
+  height: 100%;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  z-index: 2000;
+}
+
+.sk-chase {
+  width: 60px;
+  height: 60px;
+  position: relative;
+  animation: sk-chase 2.5s infinite linear both;
+}
+
+.sk-chase-dot {
+  width: 100%;
+  height: 100%;
+  position: absolute;
+  left: 0;
+  top: 0; 
+  animation: sk-chase-dot 2.0s infinite ease-in-out both; 
+}
+
+.sk-chase-dot:before {
+  content: '';
+  display: block;
+  width: 25%;
+  height: 25%;
+  background-color: #fff;
+  border-radius: 100%;
+  animation: sk-chase-dot-before 2.0s infinite ease-in-out both; 
+}
+
+.sk-chase-dot:nth-child(1) { animation-delay: -1.1s; }
+.sk-chase-dot:nth-child(2) { animation-delay: -1.0s; }
+.sk-chase-dot:nth-child(3) { animation-delay: -0.9s; }
+.sk-chase-dot:nth-child(4) { animation-delay: -0.8s; }
+.sk-chase-dot:nth-child(5) { animation-delay: -0.7s; }
+.sk-chase-dot:nth-child(6) { animation-delay: -0.6s; }
+.sk-chase-dot:nth-child(1):before { animation-delay: -1.1s; }
+.sk-chase-dot:nth-child(2):before { animation-delay: -1.0s; }
+.sk-chase-dot:nth-child(3):before { animation-delay: -0.9s; }
+.sk-chase-dot:nth-child(4):before { animation-delay: -0.8s; }
+.sk-chase-dot:nth-child(5):before { animation-delay: -0.7s; }
+.sk-chase-dot:nth-child(6):before { animation-delay: -0.6s; }
+
+@keyframes sk-chase {
+  100% { transform: rotate(360deg); } 
+}
+
+@keyframes sk-chase-dot {
+  80%, 100% { transform: rotate(360deg); } 
+}
+
+@keyframes sk-chase-dot-before {
+  50% {
+    transform: scale(0.4); 
+  } 100%, 0% {
+    transform: scale(1.0); 
+  } 
+}

vripper-ui/src/app/app.component.ts

@@ -1,185 +1,48 @@
 import { AppService } from './app.service';
 import { ClipboardService } from './clipboard.service';
 import { ElectronService } from 'ngx-electron';
-import { SettingsComponent } from './settings/settings.component';
-import { Component, OnInit, OnDestroy, NgZone, Renderer2, AfterViewInit } from '@angular/core';
-import { MatDialog, MatSnackBar } from '@angular/material';
-import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout';
-import { Observable, Subscription } from 'rxjs';
+import { Component, OnDestroy, AfterViewInit, Renderer2, ChangeDetectionStrategy, NgZone } from '@angular/core';
+import { MatDialog } from '@angular/material';
+import { Subscription, BehaviorSubject, Subject, merge } from 'rxjs';
 import { WsConnectionService, WSState } from './ws-connection.service';
-import { LoggedUser } from './common/logged-user.model';
-import { WsHandler } from './ws-handler';
-import { CMD } from './common/cmd.enum';
-import { WSMessage } from './common/ws-message.model';
-import { HttpClient } from '@angular/common/http';
-import { RemoveAllResponse } from './common/remove-all-response.model';
-import { ServerService } from './server-service';
-import { ConfirmDialogComponent } from './common/confirmation-component/confirmation-dialog';
-import { filter, flatMap } from 'rxjs/operators';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
-  styleUrls: ['./app.component.scss']
+  styleUrls: ['./app.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
-export class AppComponent implements OnInit, OnDestroy, AfterViewInit {
+export class AppComponent implements OnDestroy, AfterViewInit {
   constructor(
-    public dialog: MatDialog,
-    private breakpointObserver: BreakpointObserver,
+    private dialog: MatDialog,
     private ws: WsConnectionService,
     public electronService: ElectronService,
     private clipboardService: ClipboardService,
-    private ngZone: NgZone,
-    private httpClient: HttpClient,
-    private serverService: ServerService,
-    private _snackBar: MatSnackBar,
+    private appService: AppService,
     private renderer: Renderer2,
-    private appService: AppService
+    private ngZone: NgZone
   ) {
-    this.websocketHandlerPromise = this.ws.getConnection();
+    this.electron = new BehaviorSubject(electronService.isElectronApp);
   }
 
-  subscriptions: Subscription[] = [];
-  websocketHandlerPromise: Promise<WsHandler>;
-  currentState: WSState;
-  themeLoaded = false;
-  loggedUser: LoggedUser = new LoggedUser(null);
-  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
+  private subscriptions: Subscription[] = [];
+  appState: Subject<string> = new BehaviorSubject('CONNECTING');
+  electron: Subject<boolean>;
 
   ngAfterViewInit() {
     this.appService.renderer = this.renderer;
-  }
-
-  scan() {
-    this.appService.scan();
-  }
-
-  openSettings(): void {
-    const dialogRef = this.dialog.open(SettingsComponent, {
-      width: '70%',
-      height: '70%',
-      maxWidth: '100vw',
-      maxHeight: '100vh'
-    });
-
-    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
-      if (result.matches) {
-        dialogRef.updateSize('100%', '100%');
-      } else {
-        dialogRef.updateSize('70%', '70%');
-      }
-    });
-
-    dialogRef.afterClosed().subscribe(result => {
-      smallDialogSubscription.unsubscribe();
-    });
-  }
-
-  noConnectionState(): boolean {
-    return this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR;
-  }
-
-  connecting(): boolean {
-    return (this.currentState === WSState.INIT || this.currentState === WSState.CONNECTING) && !this.themeLoaded ;
-  }
-
-  ngOnInit() {
-    this.websocketHandlerPromise.then((handler: WsHandler) => {
-      console.log('Connecting to user stream');
-      this.subscriptions.push(
-        handler.subscribeForUser((e: LoggedUser[]) => {
-          this.ngZone.run(() => {
-            this.loggedUser = e[0];
-          });
-        })
-      );
-      handler.send(new WSMessage(CMD.USER_SUB.toString()));
-    });
-    this.currentState = WSState.INIT;
-    this.ws.state.subscribe(e => {
-      this.currentState = e;
-      if (this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR) {
-        this.dialog.closeAll();
-      } else if (this.currentState === WSState.OPEN) {
+    this.subscriptions.push(this.ws.state.subscribe(wsState => {
+      if (wsState === WSState.CLOSE || wsState === WSState.ERROR) {
+        setTimeout(() => this.ngZone.run(() => {
+          this.dialog.closeAll();
+          this.appState.next('DISCONNECTED');
+        }), 500);
+      } else if (wsState === WSState.OPEN) {
         this.clipboardService.init();
-        this.appService
-          .loadTheme()
-          .subscribe(() => this.ngZone.run(() => this.themeLoaded = true));
+        merge(this.appService.loadTheme(), this.appService.loadSettings())
+        .subscribe(() => this.ngZone.run(() => this.appState.next('CONNECTED')));
       }
-    });
-  }
-
-  clear() {
-    this.ngZone.run(() => {
-      this.httpClient.post<RemoveAllResponse>(this.serverService.baseUrl + '/post/clear/all', {}).subscribe(
-        data => {
-          this._snackBar.open(`${data.removed} items cleared`, null, { duration: 5000 });
-        },
-        error => {
-          this._snackBar.open(error.error, null, {
-            duration: 5000
-          });
-        }
-      );
-    });
-  }
-
-  remove() {
-    this.ngZone.run(() => {
-      this.dialog
-        .open(ConfirmDialogComponent, {
-          maxHeight: '100vh',
-          maxWidth: '100vw',
-          height: '200px',
-          width: '60%',
-          data: { header: 'Confirmation', content: 'Are you sure you want to remove all items ?' }
-        })
-        .afterClosed()
-        .pipe(
-          filter(e => e === 'yes'),
-          flatMap(e => this.httpClient.post<RemoveAllResponse>(this.serverService.baseUrl + '/post/remove/all', {}))
-        )
-        .subscribe(
-          data => {
-            this._snackBar.open(`${data.removed} items removed`, null, { duration: 5000 });
-          },
-          error => {
-            this._snackBar.open(error.error, null, {
-              duration: 5000
-            });
-          }
-        );
-    });
-  }
-
-  stopAll() {
-    this.ngZone.run(() => {
-      this.httpClient.post(this.serverService.baseUrl + '/post/stop/all', {}).subscribe(
-        () => {
-          this._snackBar.open(`Download stopped`, null, { duration: 5000 });
-        },
-        error => {
-          this._snackBar.open(error.error, null, {
-            duration: 5000
-          });
-        }
-      );
-    });
-  }
-
-  restartAll() {
-    this.ngZone.run(() => {
-      this.httpClient.post(this.serverService.baseUrl + '/post/restart/all', {}).subscribe(
-        () => {
-          this._snackBar.open(`Download started`, null, { duration: 5000 });
-        },
-        error => {
-          this._snackBar.open(error.error, null, {
-            duration: 5000
-          });
-        }
-      );
-    });
+    }));
   }
 
   ngOnDestroy() {

vripper-ui/src/app/app.module.ts

@@ -1,3 +1,7 @@
+import { LinkCollectorService } from './link-collector.service';
+import { ContextMenuService } from './ctxt-menu.service';
+import { PostContextMenuComponent } from './posts/post-context-menu.component';
+import { PostsDataService } from './posts-data.service';
 import { AppPreviewDirective } from './common/preview-tooltip.directive';
 import { WsConnectionService } from './ws-connection.service';
 import { AppService } from './app.service';
@@ -31,6 +35,12 @@ import { UrlRendererComponent } from './multi-post/url-renderer.component';
 import { FilterComponent } from './filter/filter.component';
 import { ScanComponent } from './scan/scan.component';
 import { StatusBarComponent } from './status-bar/status-bar.component';
+import { SelectionService } from './selection-service';
+import { ToolbarComponent } from './toolbar/toolbar.component';
+import { GrabQueueComponent } from './grab-queue/grab-queue.component';
+import { UrlGrabRendererComponent } from './grab-queue/url-renderer.component';
+import { GalleryComponent } from './gallery/gallery.component';
+import { PhotoSwipeComponent } from './photo-swipe/photo-swipe.component';
 
 @NgModule({
   declarations: [
@@ -47,17 +57,25 @@ import { StatusBarComponent } from './status-bar/status-bar.component';
     AppPreviewComponent,
     AppPreviewDirective,
     UrlRendererComponent,
+    UrlGrabRendererComponent,
     FilterComponent,
     ScanComponent,
-    StatusBarComponent
+    StatusBarComponent,
+    ToolbarComponent,
+    GrabQueueComponent,
+    PostContextMenuComponent,
+    GalleryComponent,
+    PhotoSwipeComponent
   ],
   entryComponents: [
     PostDetailComponent,
     SettingsComponent,
     ConfirmDialogComponent,
-    // MultiPostComponent,
     AppPreviewComponent,
-    ScanComponent
+    ScanComponent,
+    MultiPostComponent,
+    PostContextMenuComponent,
+    GalleryComponent
   ],
   imports: [
     BrowserAnimationsModule,
@@ -71,7 +89,8 @@ import { StatusBarComponent } from './status-bar/status-bar.component';
     AgGridModule.withComponents([
       PostProgressRendererComponent,
       PostDetailsProgressRendererComponent,
-      UrlRendererComponent
+      UrlRendererComponent,
+      UrlGrabRendererComponent
     ]),
     OverlayModule
   ],
@@ -80,7 +99,11 @@ import { StatusBarComponent } from './status-bar/status-bar.component';
     WsConnectionService,
     { provide: HTTP_INTERCEPTORS, useClass: XhrInterceptorService, multi: true },
     ServerService,
-    SharedService
+    SharedService,
+    SelectionService,
+    PostsDataService,
+    ContextMenuService,
+    LinkCollectorService
   ],
   bootstrap: [AppComponent]
 })

vripper-ui/src/app/app.service.ts

@@ -1,11 +1,14 @@
+import { Settings } from './common/settings.model';
+import { environment } from 'src/environments/environment';
 import { MatDialog } from '@angular/material';
 import { HttpClient } from '@angular/common/http';
-import { Injectable, Renderer2, NgZone } from '@angular/core';
+import { Injectable, Renderer2 } from '@angular/core';
 import { ServerService } from './server-service';
 import { tap } from 'rxjs/operators';
 import { ScanComponent } from './scan/scan.component';
-import { Observable } from 'rxjs';
+import { Observable, Subject } from 'rxjs';
 import { BreakpointState, Breakpoints, BreakpointObserver } from '@angular/cdk/layout';
+import { Title } from '@angular/platform-browser';
 
 @Injectable()
 export class AppService {
@@ -14,20 +17,34 @@ export class AppService {
     private serverService: ServerService,
     public dialog: MatDialog,
     private breakpointObserver: BreakpointObserver,
-  ) {}
+    private titleService: Title
+  ) {
+    this.titleService.setTitle('VRipper ' + environment.version);
+  }
+
+  private _darkTheme = false;
+  private _settings: Settings;
+  private _renderer: Renderer2;
+
+  // private settings$: Subject<Settings> = new Subject();
+
+  get settings(): Settings {
+    return { ...this._settings };
+  }
+
+  get darkTheme(): boolean {
+    return this._darkTheme;
+  }
 
-  darkTheme = false;
-  _renderer: Renderer2;
   set renderer(renderer: Renderer2) {
     this._renderer = renderer;
   }
 
   isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
-  isScanOpen = false;
 
   updateTheme(darkTheme: boolean) {
-    this.darkTheme = darkTheme;
-    if (this.darkTheme) {
+    this._darkTheme = darkTheme;
+    if (this._darkTheme) {
       this._renderer.addClass(document.body, 'dark-theme');
       this._renderer.removeClass(document.body, 'light-theme');
     } else {
@@ -36,17 +53,27 @@ export class AppService {
     }
     this.httpClient
       .post<Theme>(this.serverService.baseUrl + '/settings/theme', {
-        darkTheme: this.darkTheme
+        darkTheme: this._darkTheme
       })
       .subscribe();
   }
 
+  updateSettings(settings: Settings) {
+    this._settings = settings;
+  }
+
   loadTheme() {
     return this.httpClient
       .get<Theme>(this.serverService.baseUrl + '/settings/theme')
       .pipe(tap(theme => this.updateTheme(theme.darkTheme)));
   }
 
+  loadSettings() {
+    return this.httpClient
+      .get<Settings>(this.serverService.baseUrl + '/settings')
+      .pipe(tap(settings => this.updateSettings(settings)));
+  }
+
   scan(url?: string) {
 
     const scanDialog = this.dialog.open(ScanComponent, {
@@ -65,11 +92,8 @@ export class AppService {
       }
     });
 
-    scanDialog.afterOpened().subscribe(() => this.isScanOpen = true);
-
     scanDialog.afterClosed().subscribe(() => {
       smallDialogSubscription.unsubscribe();
-      this.isScanOpen = false;
     });
   }
 }

vripper-ui/src/app/common/cmd.enum.ts

@@ -5,10 +5,12 @@ export enum CMD {
     SPEED_SUB = 'SPEED_SUB',
     USER_SUB = 'USER_SUB',
     THREAD_PARSING_SUB = 'THREAD_PARSING_SUB',
+    GRAB_QUEUE_SUB = 'GRAB_QUEUE_SUB',
     POSTS_UNSUB = 'POSTS_UNSUB',
     POST_DETAILS_UNSUB = 'POST_DETAILS_UNSUB',
     GLOBAL_STATE_UNSUB = 'GLOBAL_STATE_UNSUB',
     SPEED_UNSUB = 'SPEED_UNSUB',
     USER_UNSUB = 'USER_UNSUB',
-    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB'
+    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB',
+    GRAB_QUEUE_UNSUB = 'GRAB_QUEUE_UNSUB'
 }

vripper-ui/src/app/common/preview-tooltip.component.ts

@@ -4,11 +4,10 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
 @Component({
   selector: 'app-preview',
   template: `
-  <div class="previews" style="display: inline-block; white-space: nowrap;">
+  <div class="previews" style="display: inline-block; white-space: nowrap; height: 200px; max-width: 400px">
     <ng-container *ngFor="let link of links">
       <img
         [@simpleFadeAnimation]="'in'"
-        class="mat-elevation-z8"
         [src]="link"
       />
     </ng-container>
@@ -18,9 +17,8 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
     `
       img {
         max-height: 200px;
-        max-width: 200px;
+        max-width: 400px;
         margin-left: 5px;
-        border: 1px solid #900000;
       }
     `
   ],

vripper-ui/src/app/common/preview-tooltip.directive.ts

@@ -1,30 +1,48 @@
 import { AppPreviewComponent } from './preview-tooltip.component';
-import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone } from '@angular/core';
+import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone, OnDestroy } from '@angular/core';
 import { OverlayRef, Overlay, OverlayPositionBuilder } from '@angular/cdk/overlay';
 import { ComponentPortal } from '@angular/cdk/portal';
 
 @Directive({ selector: '[appPreview]' })
-export class AppPreviewDirective implements OnInit {
+export class AppPreviewDirective implements OnInit, OnDestroy {
   private overlayRef: OverlayRef;
   tooltipPortal = new ComponentPortal(AppPreviewComponent);
 
   @Input() appPreview: string[];
 
-  constructor(private overlayPositionBuilder: OverlayPositionBuilder,
+  constructor(
+    private overlayPositionBuilder: OverlayPositionBuilder,
     private elementRef: ElementRef,
     private overlay: Overlay,
-    private ngZone: NgZone) {}
+    private ngZone: NgZone
+  ) {}
 
   ngOnInit() {
     const positionStrategy = this.overlayPositionBuilder
-    .flexibleConnectedTo(this.elementRef)
-    .withPositions([{
-      originX: 'center',
-      originY: 'top',
-      overlayX: 'center',
-      overlayY: 'bottom',
-    }]);
-    this.overlayRef = this.overlay.create({ positionStrategy });
+      .flexibleConnectedTo(this.elementRef)
+      .withPush(true)
+      .withGrowAfterOpen(true)
+      .withPositions([
+        {
+          originX: 'center',
+          originY: 'bottom',
+          overlayX: 'center',
+          overlayY: 'top'
+        },
+        {
+          originX: 'center',
+          originY: 'top',
+          overlayX: 'center',
+          overlayY: 'bottom'
+        }
+      ]);
+    this.overlayRef = this.overlay.create({ positionStrategy, scrollStrategy: this.overlay.scrollStrategies.close() });
+  }
+
+  ngOnDestroy() {
+    this.ngZone.run(() => {
+      this.overlayRef.dispose();
+    });
   }
 
   @HostListener('mouseenter')

vripper-ui/src/app/common/settings.model.ts

@@ -1,10 +1,12 @@
 export interface Settings {
     downloadPath: string;
     maxThreads: number;
+    maxTotalThreads: number;
     autoStart: boolean;
     vLogin: boolean;
     vUsername: string;
     vPassword: string;
     vThanks: boolean;
     desktopClipboard: boolean;
+    viewPhotos: boolean;
   }

vripper-ui/src/app/common/vr-post-parse.model.ts

@@ -6,10 +6,7 @@ export class VRPostParse {
     public title: string,
     public imageCount: number,
     public url: string,
-    public previews: string[]
+    public previews: string[],
+    public hosts: string
   ) {}
 }
-
-export class VRThreadParseState {
-  constructor(public threadId: string, public state: string) {}
-}

vripper-ui/src/app/ctxt-menu.service.ts

@@ -0,0 +1,68 @@
+import { PostState } from './posts/post-state.model';
+import { Injectable, ComponentRef } from '@angular/core';
+import { filter, take } from 'rxjs/operators';
+import { fromEvent, Subscription } from 'rxjs';
+import { OverlayPositionBuilder, Overlay, OverlayRef } from '@angular/cdk/overlay';
+import { ComponentPortal } from '@angular/cdk/portal';
+import { PostContextMenuComponent } from './posts/post-context-menu.component';
+
+@Injectable()
+export class ContextMenuService {
+  constructor(private overlayPositionBuilder: OverlayPositionBuilder, private overlay: Overlay) {}
+
+  postCtxtMenuPortal = new ComponentPortal(PostContextMenuComponent);
+  private postCtxtMenuOverlayRef: OverlayRef;
+  sub: Subscription;
+
+  openPostCtxtMenu(mouseEvent: MouseEvent, postState: PostState) {
+    mouseEvent.preventDefault();
+    this.closePostCtxtMenu();
+    const positionStrategy = this.overlayPositionBuilder
+      .flexibleConnectedTo({ x: mouseEvent.x, y: mouseEvent.y })
+      .withPush(true)
+      .withGrowAfterOpen(true)
+      .withPositions([
+        {
+          originX: 'start',
+          originY: 'bottom',
+          overlayX: 'start',
+          overlayY: 'top'
+        },
+        {
+          originX: 'start',
+          originY: 'top',
+          overlayX: 'start',
+          overlayY: 'bottom'
+        }
+      ]);
+    this.postCtxtMenuOverlayRef = this.overlay.create({
+      positionStrategy
+    });
+    const ref: ComponentRef<PostContextMenuComponent> = this.postCtxtMenuOverlayRef.attach(this.postCtxtMenuPortal);
+    ref.instance.postState = postState;
+    ref.instance.contextMenuService = this;
+    this.sub = fromEvent<MouseEvent>(document, 'click')
+      .pipe(
+        filter(event => {
+          const clickTarget = event.target as HTMLElement;
+          return !!this.postCtxtMenuOverlayRef && !this.postCtxtMenuOverlayRef.overlayElement.contains(clickTarget);
+        }),
+        take(1)
+      )
+      .subscribe(() => this.closePostCtxtMenu());
+  }
+
+  closePostCtxtMenu() {
+    if (this.sub != null) {
+      this.sub.unsubscribe();
+    }
+    if (this.postCtxtMenuOverlayRef) {
+      this.postCtxtMenuOverlayRef.dispose();
+      this.postCtxtMenuOverlayRef = null;
+    }
+  }
+}
+
+export interface Theme {
+  darkTheme: boolean;
+}

vripper-ui/src/app/filter/filter.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { FilterComponent } from './filter.component';
-
-describe('FilterComponent', () => {
-  let component: FilterComponent;
-  let fixture: ComponentFixture<FilterComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ FilterComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(FilterComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

vripper-ui/src/app/gallery/gallery.component.html

@@ -0,0 +1,25 @@
+<div class="dialog-container" fxLayout="column" style="height: 100%;">
+  <div fxFlex="nogrow">
+    <h2 class="no-wrap" mat-dialog-title>{{ dialogData.title }}</h2>
+  </div>
+  <mat-dialog-content fxFlex="grow">
+    <div class="container">
+      <div class="masonry">
+        <ng-container *ngFor="let img of images | async; let i=index">
+          <figure [style.opacity]="img._initialized ? 1 : 0"
+                  [style.visibility]="img._initialized ? 'visible' : 'hidden'" class="masonry-brick">
+            <img (click)="openSlideshow(i)" (load)="loaded(img)" [src]="img.msrc"/>
+          </figure>
+        </ng-container>
+
+        <app-photo-swipe #photoSwipe></app-photo-swipe>
+      </div>
+    </div>
+  </mat-dialog-content>
+  <mat-dialog-actions align="end" fxFlex="nogrow">
+    <button (click)="refresh()" [disabled]="(disableRefresh | async)" color="primary" mat-raised-button>
+      <mat-icon>refresh</mat-icon>
+      <span>Refresh</span></button>
+    <button mat-dialog-close mat-raised-button>Close</button>
+  </mat-dialog-actions>
+</div>

vripper-ui/src/app/gallery/gallery.component.scss

@@ -0,0 +1,22 @@
+.container {
+  height: 100%;
+}
+
+.masonry {
+  width: 100%;
+}
+
+.masonry-brick {
+  height: 250px;
+  width: 250px;
+  display: inline-block;
+  cursor: pointer;
+  margin: 0 3px;
+  transition: opacity 0.5s ease-in, visibility 0.5s;
+}
+
+img {
+  object-fit: cover;
+  height: 250px;
+  width: 250px;
+}

vripper-ui/src/app/gallery/gallery.component.ts

@@ -0,0 +1,98 @@
+import { IImage, PhotoSwipeComponent } from './../photo-swipe/photo-swipe.component';
+import { Component, OnInit, ChangeDetectionStrategy, Inject, NgZone, OnDestroy, ViewChild } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { MatDialogRef, MAT_DIALOG_DATA, MatSnackBar } from '@angular/material';
+import { ServerService } from '../server-service';
+import { PostState } from '../posts/post-state.model';
+import { BehaviorSubject, Subject } from 'rxjs';
+import { finalize } from 'rxjs/operators';
+
+class Image extends IImage {
+  constructor(
+    public title: string,
+    public src: string,
+    public msrc: string,
+    public w: number,
+    public h: number,
+    public _initialized: boolean
+  ) {
+    super(src, msrc, w, h);
+  }
+}
+
+@Component({
+  selector: 'app-gallery',
+  templateUrl: './gallery.component.html',
+  styleUrls: ['./gallery.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class GalleryComponent implements OnInit, OnDestroy {
+  constructor(
+    public dialogRef: MatDialogRef<GalleryComponent>,
+    @Inject(MAT_DIALOG_DATA) public dialogData: PostState,
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    private _snackBar: MatSnackBar,
+    private ngZone: NgZone
+  ) {}
+
+  images: Subject<Image[]> = new BehaviorSubject([]);
+  _images: Image[] = [];
+  disableRefresh: Subject<boolean> = new BehaviorSubject(true);
+
+  @ViewChild('photoSwipe', { static: true }) photoSwipe: PhotoSwipeComponent;
+
+  loaded(img: Image) {
+    img._initialized = true;
+  }
+
+  ngOnInit() {
+    this.refresh();
+  }
+
+  ngOnDestroy() {}
+
+  openSlideshow(index: number) {
+    this.photoSwipe.openGallery(this._images, {
+      index: index,
+      getThumbBoundsFn: this.getThumbBoundsFn,
+      showHideOpacity: true,
+      preload: [1, 8]
+    });
+  }
+
+  getThumbBoundsFn(index: number) {
+    // See Options -> getThumbBoundsFn section of documentation for more info
+    const thumbnail = document.querySelectorAll('.masonry-brick img')[index],
+      pageYScroll = window.pageYOffset || document.documentElement.scrollTop,
+      rect = thumbnail.getBoundingClientRect();
+
+    return { x: rect.left, y: rect.top + pageYScroll, w: rect.width };
+  }
+
+  refresh() {
+    this.httpClient.get<Image[]>(this.serverService.baseUrl + '/gallery/' + this.dialogData.postId)
+    .pipe(finalize(() => this.disableRefresh.next(false)))
+    .subscribe(
+      response => {
+        this.ngZone.run(() => {
+          response.forEach(i => {
+            i.src = this.serverService.baseUrl + '/image/' + this.dialogData.postId + '/' + i.src;
+            i.msrc = this.serverService.baseUrl + '/image/thumb/' + this.dialogData.postId + '/' + i.msrc;
+            i._initialized = false;
+          });
+          this._images = response;
+          this.images.next(this._images);
+        });
+      },
+      error => {
+        this.ngZone.run(() => {
+          this.dialogRef.close();
+          this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+            duration: 5000
+          });
+        });
+      }
+    );
+  }
+}

vripper-ui/src/app/grab-queue/grab-queue.component.html

@@ -0,0 +1,2 @@
+<ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-material" style="width: 100%; height: 100%;">
+</ag-grid-angular>

vripper-ui/src/app/grab-queue/grab-queue.component.ts

@@ -0,0 +1,53 @@
+import { LinkCollectorService } from './../link-collector.service';
+import { UrlGrabRendererComponent } from './url-renderer.component';
+import { GrabQueueDataSource } from './grab-queue.datasource';
+import { Component, OnInit, NgZone, ChangeDetectionStrategy } from '@angular/core';
+import { GridOptions } from 'ag-grid-community';
+import { WsConnectionService } from '../ws-connection.service';
+
+@Component({
+  selector: 'app-grab-queue',
+  templateUrl: './grab-queue.component.html',
+  styleUrls: ['./grab-queue.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class GrabQueueComponent implements OnInit {
+  constructor(private wsConnection: WsConnectionService, private zone: NgZone, private linkCollectorService: LinkCollectorService) {
+    this.gridOptions = <GridOptions>{
+      columnDefs: [
+        {
+          headerName: 'Url',
+          field: 'link',
+          sortable: true,
+          sort: 'asc',
+          suppressMovable: true,
+          cellRenderer: 'urlCellRenderer'
+        }
+      ],
+      rowHeight: 48,
+      animateRows: true,
+      rowData: [],
+      frameworkComponents: {
+        urlCellRenderer: UrlGrabRendererComponent
+      },
+      overlayLoadingTemplate: '<span></span>',
+      overlayNoRowsTemplate: '<span></span>',
+      getRowNodeId: data => data['link'],
+      onGridReady: () => {
+        this.gridOptions.api.sizeColumnsToFit();
+        this.dataSource = new GrabQueueDataSource(this.wsConnection, this.gridOptions, this.zone);
+        this.dataSource.connect();
+      },
+      onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
+      onRowDataUpdated: () => {
+        this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount());
+        this.gridOptions.api.sizeColumnsToFit();
+      }
+    };
+  }
+
+  gridOptions: GridOptions;
+  dataSource: GrabQueueDataSource;
+
+  ngOnInit() {}
+}

vripper-ui/src/app/grab-queue/grab-queue.datasource.ts

@@ -0,0 +1,60 @@
+import { GrabQueueState } from './grab-queue.model';
+import { CMD } from './../common/cmd.enum';
+import { WSMessage } from './../common/ws-message.model';
+import { Subscription } from 'rxjs';
+import { WsConnectionService } from '../ws-connection.service';
+import { GridOptions } from 'ag-grid-community';
+import { WsHandler } from '../ws-handler';
+import { NgZone } from '@angular/core';
+
+export class GrabQueueDataSource {
+  constructor(
+    private wsConnectionService: WsConnectionService,
+    private gridOptions: GridOptions,
+    private zone: NgZone
+  ) {
+    this.websocketHandlerPromise = this.wsConnectionService.getConnection();
+  }
+
+  websocketHandlerPromise: Promise<WsHandler>;
+  subscriptions: Subscription[] = [];
+
+  connect() {
+    this.websocketHandlerPromise.then((handler: WsHandler) => {
+      console.log('Connecting to grab queue datasource');
+      this.subscriptions.push(
+        handler.subscribeForGrabQueue((e: GrabQueueState[]) => {
+          this.zone.run(() => {
+            const toAdd = [];
+            const toUpdate = [];
+            const toRemove = [];
+            e.forEach(v => {
+              if (v.removed) {
+                if (this.gridOptions.api.getRowNode(v.link) != null) {
+                  toRemove.push(this.gridOptions.api.getRowNode(v.link).data);
+                }
+                return;
+              }
+
+              if (this.gridOptions.api.getRowNode(v.link) == null) {
+                toAdd.push(v);
+              } else {
+                toUpdate.push(v);
+              }
+            });
+            this.gridOptions.api.updateRowData({ update: toUpdate, add: toAdd, remove: toRemove });
+          });
+        })
+      );
+      handler.send(new WSMessage(CMD.GRAB_QUEUE_SUB.toString()));
+    });
+  }
+
+  disconnect() {
+    console.log('Disconnecting from grab queue datasource');
+    this.subscriptions.forEach(e => e.unsubscribe());
+    this.websocketHandlerPromise.then((handler: WsHandler) => {
+      handler.send(new WSMessage(CMD.GRAB_QUEUE_UNSUB.toString()));
+    });
+  }
+}

vripper-ui/src/app/grab-queue/grab-queue.model.ts

@@ -0,0 +1,9 @@
+export class GrabQueueState {
+  constructor(
+    public type: string,
+    public link: string,
+    public threadId: string,
+    public postId: string,
+    public removed: boolean
+  ) {}
+}

vripper-ui/src/app/grab-queue/url-renderer.component.ts

@@ -0,0 +1,98 @@
+import { ElectronService } from 'ngx-electron';
+import { MultiPostComponent } from './../multi-post/multi-post.component';
+import { OnInit, OnDestroy, Component, ChangeDetectionStrategy } from '@angular/core';
+import { AgRendererComponent } from 'ag-grid-angular';
+import { ICellRendererParams } from 'ag-grid-community';
+import { GrabQueueState } from './grab-queue.model';
+import { MatDialog, MatSnackBar } from '@angular/material';
+import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout';
+import { Observable } from 'rxjs';
+import { ServerService } from '../server-service';
+import { HttpClient } from '@angular/common/http';
+
+@Component({
+  selector: 'app-url-cell-grab',
+  template: `
+    <div fxLayout="row" fxLayoutAlign="space-between center">
+      <span style="white-space: nowrap; overflow: hidden; text-overflow: ellipsis;"
+        ><a (click)="goTo()" href="javascript:void(0)">{{ grabQueue.link }}</a></span
+      >
+      <span>
+        <button style="margin-right: 5px" (click)="grab()" color="primary" mat-mini-fab>
+          <mat-icon>get_app</mat-icon>
+        </button>
+        <button (click)="remove()" color="primary" mat-mini-fab><mat-icon>delete</mat-icon></button>
+      </span>
+    </div>
+  `,
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class UrlGrabRendererComponent implements OnInit, OnDestroy, AgRendererComponent {
+  constructor(
+    public dialog: MatDialog,
+    private breakpointObserver: BreakpointObserver,
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    private _snackBar: MatSnackBar,
+    private electronService: ElectronService
+  ) {}
+
+  grabQueue: GrabQueueState;
+  params: ICellRendererParams;
+  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
+
+  ngOnInit(): void {}
+
+  ngOnDestroy(): void {}
+
+  agInit(params: ICellRendererParams): void {
+    this.grabQueue = params.data;
+  }
+
+  refresh(params: ICellRendererParams): boolean {
+    return false;
+  }
+
+  grab() {
+    const dialogRef = this.dialog.open(MultiPostComponent, {
+      width: '90%',
+      height: '90%',
+      maxWidth: '100vw',
+      maxHeight: '100vh',
+      data: this.grabQueue
+    });
+
+    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
+      if (result.matches) {
+        dialogRef.updateSize('100%', '100%');
+      } else {
+        dialogRef.updateSize('90%', '90%');
+      }
+    });
+
+    dialogRef.afterClosed().subscribe(() => {
+      if (smallDialogSubscription != null) {
+        smallDialogSubscription.unsubscribe();
+      }
+    });
+  }
+
+  goTo() {
+    if (this.electronService.isElectronApp) {
+      this.electronService.shell.openExternal(this.grabQueue.link);
+    } else {
+      window.open(this.grabQueue.link, '_blank');
+    }
+  }
+
+  remove() {
+    this.httpClient.post(this.serverService.baseUrl + '/grab/remove', { url: this.grabQueue.link }).subscribe(
+      () => {},
+      error => {
+        this._snackBar.open(error.error || 'Unexpected error, check log file', null, {
+          duration: 5000
+        });
+      }
+    );
+  }
+}

vripper-ui/src/app/home/home.component.html

@@ -1,10 +1,14 @@
-<div *ngIf="loading" class="overlay loading" fxLayout="row" fxLayoutAlign="center center">
-  <mat-progress-spinner [mode]="spinnerMode" [value]="spinnerValue" style="position: absolute;"></mat-progress-spinner>
-</div>
-<div fxLayout="column" fxLayoutGap="10px" style="height: 100%">
-  <div fxFlex="0 0 calc(100% - 90px)">
-    <app-posts
-      style="width: 100%; height: 100%;"
-    ></app-posts>
-  </div>
-</div>
+<mat-tab-group dynamicHeight style="height: 100%; margin-bottom: 20px; overflow: hidden;">
+  <mat-tab label="Downloads">
+    <app-posts style="width: 100%; height: 100%;"></app-posts>
+  </mat-tab>
+  <mat-tab>
+    <ng-template mat-tab-label>
+      <span style="margin-right: 5px;">Link Collector</span>
+      <ng-container *ngIf="linkCollectorService.count | async as count">
+        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count">link</mat-icon>
+      </ng-container>
+    </ng-template>
+    <app-grab-queue style="width: 100%; height: 100%;"></app-grab-queue>
+  </mat-tab>
+</mat-tab-group>