v2.10.4

Add system notification
Bug fixes

.gitignore

@@ -2,4 +2,5 @@
 /**/*/node_modules
 /**/*/target
 /**/*/build-dir
+/**/*/java-runtime
 .idea

CHANGELOG.md

@@ -1,5 +1,209 @@
 # Changelog
 
+## [2.10.4] - 2020-01-18
+### Changed
+- Zero value for total max download will disable the limit
+- Add system notification
+- Bug fixes
+
+## [2.10.3] - 2020-01-16
+### Changed
+- Some bug fixes
+- Enhance download queue logic
+- Clear cache button
+- Fix the partial status
+- Sanitize URLs with spaces
+### Added
+- Support for Postimg host
+- Support for Imagevenue host
+
+## [2.10.2] - 2019-12-29
+### Changed
+- Upgrade electron to v7
+- Bug fix
+
+## [2.10.1] - 2019-12-27
+### Changed
+- Bug fix
+
+## [2.10.0] - 2019-12-27
+### Changed
+- Fix spring dependency bug
+### Added
+- Add support for imgspice
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Fix build issue
+
+## [2.9.0] - 2019-12-27
+### Changed
+- Decrease timeout for fast fail
+### Added
+- Allow many hosts to run at the same time
+- Add title to gallery
+- Add global concurrent downloads
+- Add counter badge in link collector
+
+## [2.8.1] - 2019-12-24
+### Changed
+- Fix issue with path creation
+
+## [2.8.0] - 2019-12-24
+### Added
+- Photo gallery feature
+
+## [2.7.1] - 2019-12-21
+### Added
+- Add version to title bar
+
+## [2.7.0] - 2019-12-21
+### Changed
+- Add missing icon
+### Added
+- Add support for pixxxels
+
+## [2.6.0] - 2019-12-21
+### Changed
+- Disable download button when nothing selected in Link collector grab menu
+- Better quality app icon
+- Implement context menu on download tab
+- Add stop remove and start button to context menu
+- Link will not be removed automatically from link collector tab
+- Some UI fixes
+
+## [2.5.1] - 2019-12-09
+### Changed
+- Disable download button when nothing selected in Link collector grab menu
+### Added
+- Add hosts column in Link collector grab menu
+
+## [2.5.0] - 2019-12-08
+### Changed
+- Change loading screen
+- UI Fixes
+- Fix bug with previews
+### Added
+- Add link collector feature
+- Add support for multiple links at a time
+
+## [2.4.1] - 2019-11-26
+### Changed
+- Increase attempts for establishing ws connection between front and back end
+- Change default download folder to Home folder (installer), App folder (portable)
+
+## [2.4.0] - 2019-11-23
+### Changed
+- Pimpandhost Host support
+
+## [2.3.2] - 2019-11-23
+### Changed
+- Fix a bug with the scan component
+
+## [2.3.1] - 2019-11-22
+### Changed
+- Fix connexion reset issue
+- Fix Post Status when autostart option is selected
+- Add stop all button
+
+## [2.3.0] - 2019-11-21
+### Changed
+- UI enhancements and overhaul
+- Stability fixes with HTTP connections
+- MacOS official support
+- Use local sockjs and material icons font
+
+## [2.2.5] - 2019-11-09
+### Changed
+- Exclude thumbs download for imagezilla and pixhost
+- Change retry on fail logic (may help avoid having download errors)
+
+## [2.2.4] - 2019-10-05
+### Added
+- AppImage target for linux
+
+## [2.2.3] - 2019-10-01
+### Changed
+- Strip libjvm.so for debian
+
+## [2.2.2] - 2019-10-01
+### Changed
+- Update product name
+
+## [2.2.1] - 2019-10-01
+### Changed
+- Update dependency list for deb package
+
+## [2.2.0] - 2019-10-01
+### Added
+- Embedd JRE within the app
+### Changed
+- Fix 'Start' on pending items
+- Upgrade to Java 11
+
+## [2.1.0] - 2019-09-30
+### Added
+- Add a create a subfolder per thread option
+
+## [2.0.4] - 2019-09-27
+### Changed
+- Fix some issues with ImxHost
+
+## [2.0.3] - 2019-09-23
+### Changed
+- Update deb dependencies
+
+## [2.0.2] - 2019-09-23
+### Changed
+- Add portable mode
+
+## [2.0.1] - 2019-09-22
+### Changed
+- Change app name
+
+## [2.0.0] - 2019-09-22
+### Added
+- Add sub folder option
+- Add clear finished option
+### Changed
+- UI enhancements
+- Bug fixes
+- Change data and log paths
+
+## [1.6.5] - 2019-09-18
+### Changed
+- electron update
+
+## [1.6.4] - 2019-09-18
+### Changed
+- Change menu item name
+
+## [1.6.3] - 2019-09-18
+### Changed
+- Better multithread management
+- Fix partial state UI
+
+## [1.6.2] - 2019-09-17
+### Changed
+- Rework UI colors
+- Fix scan input bug
+
+## [1.6.1] - 2019-09-16
+### Changed
+- Rework UI
+- Fix clipboard monitoring
+
+## [1.6.0] - 2019-09-15
+### Changed
+- Rework the UI
+- Add a dark theme
+- Fix the force ordering bug
+- Other bug fixes
+
+## [1.5.2] - 2019-09-01
+### Added
+- imx.to host fix
+
 ## [1.5.1] - 2019-08-24
 ### Added
 - Bug fixes

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.5.1</version>
+    <version>2.10.4</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/main.js

@@ -4,7 +4,6 @@ const url = require("url");
 const getPort = require("get-port");
 const { spawn } = require("child_process");
 const { ipcMain } = require("electron");
-const commandExists = require("command-exists").sync;
 const { dialog } = require("electron");
 const axios = require('axios');
 const appDir = process.env.APPDIR;
@@ -23,27 +22,18 @@ process.on("uncaughtException", err => {
   process.exit(1);
 });
 
-if (!commandExists("java")) {
-  dialog.showErrorBox(
-    "Java command is missing",
-    "Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
-  );
-  process.exit(1);
-}
-
 function createWindow() {
   let icon;
   if(process.platform === "win32") {
-    icon = path.join(__dirname, `icon.ico`);
+    icon = __dirname + '/icon.ico';
   } else {
-    icon = path.join(__dirname, `icon.icns`);
+    icon = __dirname + '/icon.png';
   }
   win = new BrowserWindow({
     width: 1024,
     height: 800,
-    minWidth: 640,
-    minHeight: 480,
-    frame: false,
+    minWidth: 800,
+    minHeight: 600,
     webPreferences: {
       nodeIntegration: true
     },
@@ -51,6 +41,8 @@ function createWindow() {
   });
 
   win.removeMenu();
+  win.setMenu(null);
+  win.setMenuBarVisibility(false);
 
   win.loadURL(
     url.format({
@@ -67,33 +59,52 @@ function createWindow() {
   });
 }
 
-getPort().then(port => {
-  serverPort = port;
-  ipcMain.on("get-port", event => {
-    event.reply("port", port);
-  });
-  vripperServer = spawn("java", [
-    "-Xms256m",
-    "-Xmx1024m",
-    "-Dvripper.server.port=" + port,
-    "-jar",
-    appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
-    path.join(app.getPath('exe'), "../bin/vripper-server.jar")
-  ], {
-    stdio: 'ignore'
-  });
-  vripperServer.on('exit', (code, signal) => {
-    console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
-    terminated = true;
-    app.quit();
-  });
-});
-
 const gotTheLock = app.requestSingleInstanceLock();
 
 if (!gotTheLock) {
   app.quit();
 } else {
+  app.setAppUserModelId(process.execPath)
+  getPort().then(port => {
+    serverPort = port;
+    ipcMain.on("get-port", event => {
+      event.reply("port", port);
+    });
+    let javaBinPath;
+    if(appDir !== undefined) {
+      javaBinPath = path.join(appDir, "java-runtime/bin/java");
+    } else {
+      if(process.platform === 'darwin') {
+        javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+      } else {
+        javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+      }
+    }
+    let jarPath;
+    if(appDir !== undefined) {
+      jarPath = path.join(appDir, "bin/vripper-server.jar");
+    } else {
+      if(process.platform === 'darwin') {
+        jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+      } else {
+        jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+      }
+    }
+    vripperServer = spawn(javaBinPath, [
+      "-Xms256m",
+      "-Dvripper.server.port=" + port,
+      "-jar",
+      jarPath
+    ], {
+      stdio: 'ignore'
+    });
+    vripperServer.on('exit', (code, signal) => {
+      console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
+      terminated = true;
+      app.quit();
+    });
+  });
+  
   app.on("second-instance", (event, commandLine, workingDirectory) => {
     if (win) {
       if (win.isMinimized()) win.restore();
@@ -125,6 +136,7 @@ if (!gotTheLock) {
       })
       .catch((error) => {
         // Terminate immediately
+        console.log(error);
         vripperServer.kill('SIGKILL');
         terminated = true;
         app.quit();

vripper-electron/package.json

@@ -1,12 +1,14 @@
 {
   "name": "vripper-electron",
-  "version": "1.5.1",
-  "description": "",
+  "version": "2.10.4",
+  "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
-  "author": "",
+  "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
+  "homepage": "https://github.com/death-claw/vripper-project",
   "license": "ISC",
   "build": {
     "appId": "tn.mnlr.vripper",
+    "productName": "VRipper",
     "files": [
       "**/*",
       "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -29,31 +31,52 @@
       {
         "from": "../vripper-server/target/vripper-server-${version}-electron.jar",
         "to": "bin/vripper-server.jar"
+      },
+      {
+        "from": "java-runtime",
+        "to": "java-runtime"
       }
     ],
     "win": {
-      "target": "nsis"
+      "icon": "icon.ico",
+      "target": [
+        "nsis"
+      ]
+    },
+    "nsis": {
+      "createDesktopShortcut": "always",
+      "oneClick": false,
+      "perMachine": false
     },
     "linux": {
+      "synopsis": "vipergirls.to ripper",
+      "category": "Utility",
+      "packageCategory": "Utility",
+      "icon": "icons",
       "target": [
-        "zip"
+        "AppImage"
       ]
+    },
+    "mac": {
+      "category": "public.app-category.utilities",
+      "target": [
+        "dmg"
+      ],
+      "icon": "icon.icns"
     }
   },
   "scripts": {
     "start": "electron .",
-    "dist": "node pre-build.js && build"
+    "dist": "node pre-build.js && electron-builder"
   },
   "devDependencies": {
-    "electron": "^6.0.4",
-    "electron-builder": "^20.44.4"
+    "electron": "^7.1.7",
+    "electron-builder": "^21.2.0"
   },
   "dependencies": {
     "axios": "^0.19.0",
     "cheerio": "^1.0.0-rc.3",
-    "command-exists": "^1.2.8",
     "copy-dir": "^1.1.0",
-    "custom-electron-titlebar": "^3.0.10",
     "electron-context-menu": "^0.13.0",
     "get-port": "^5.0.0",
     "rimraf": "^2.6.3"

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-      <version>1.5.1</version>
+    <version>2.10.4</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-electron/pre-build.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
 const fs = require('fs');
 const rimraf = require("rimraf");
 const copydir = require("copy-dir");
-
+const { execSync } = require("child_process");
 
 rimraf.sync('./build/vripper-ui');
 copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
 
 fs.writeFileSync('./build/vripper-ui/index.html', $.html());
 
+console.log('Building runtime environment');
+rimraf.sync('java-runtime');
+execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
+if(process.platform === 'linux') {
+    console.log('Stripping libjvm.so');
+    execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
+}

vripper-electron/renderer.js

@@ -1,12 +1,3 @@
 console.log("renderer intialized");
-const customTitlebar = require("custom-electron-titlebar");
-const electron = require("electron").remote;
 const contextMenu = require("electron-context-menu");
-
-contextMenu({});
-const menu = new electron.Menu();
-const titlebar = new customTitlebar.Titlebar({
-  backgroundColor: customTitlebar.Color.fromHex("#3f51b5"),
-  menu: menu
-});
-titlebar.updateTitle("Viper Ripper");
+contextMenu({});

vripper-server/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>1.5.1</version>
+        <version>2.10.4</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
     <description>vripper-server</description>
 
     <properties>
-        <java.version>1.8</java.version>
+        <java.version>11</java.version>
     </properties>
 
     <dependencies>
@@ -67,6 +67,16 @@
             <artifactId>failsafe</artifactId>
             <version>2.0.1</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>28.1-jre</version>
+        </dependency>
+        <dependency>
+            <groupId>org.imgscalr</groupId>
+            <artifactId>imgscalr-lib</artifactId>
+            <version>4.2</version>
+        </dependency>
         <dependency>
             <groupId>tn.mnlr</groupId>
             <artifactId>vripper-ui</artifactId>

vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java

@@ -0,0 +1,24 @@
+package tn.mnlr.vripper;
+
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.event.ContextRefreshedEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+import tn.mnlr.vripper.services.PersistenceService;
+
+@Component
+public class EventListenerBean {
+
+    @Autowired
+    private PersistenceService persistenceService;
+
+    @Getter
+    private static boolean init = false;
+
+    @EventListener
+    public void onApplicationEvent(ContextRefreshedEvent event) {
+        init = true;
+        persistenceService.restore();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/SpringContext.java

@@ -17,24 +17,19 @@ public class SpringContext implements ApplicationContextAware {
 
     private static ConfigurableApplicationContext context;
 
-    /**
-     * Returns the Spring managed bean instance of the given class type (if it exists).
-     * Returns null otherwise.
-     *
-     * @param beanClass
-     * @return
-     */
-    public static <T extends Object> T getBean(Class<T> beanClass) {
+    public static <T> T getBean(Class<T> beanClass) {
         return context.getBean(beanClass);
     }
 
-    public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
+    public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
         return context.getBeansOfType(beanClass);
     }
 
     public static void close() {
         logger.info("Application terminating...");
-        context.close();
+        if (context != null) {
+            context.close();
+        }
 
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java

@@ -2,18 +2,9 @@ package tn.mnlr.vripper;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.stereotype.Component;
-import tn.mnlr.vripper.exception.VripperException;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.PersistenceService;
-import tn.mnlr.vripper.services.PostParser;
-import tn.mnlr.vripper.services.VipergirlsAuthService;
 
-import java.io.File;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -21,67 +12,15 @@ import java.util.concurrent.Executors;
 public class VripperApplication {
 
     private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
-
-    public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json";
-
     public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
     public static void main(String[] args) {
-
         try {
+            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
             SpringApplication.run(VripperApplication.class, args);
         } catch (Exception e) {
             logger.error("Failed to run the application", e);
         }
     }
-
-    @Component
-    public class AppCommandRunner implements CommandLineRunner {
-
-        @Autowired
-        private VipergirlsAuthService authService;
-
-        @Autowired
-        private PersistenceService persistenceService;
-
-        @Autowired
-        private AppSettingsService appSettingsService;
-
-        @Override
-        public void run(String... args) {
-
-            persistenceService.restore();
-            appSettingsService.restore();
-            registerShutdownHook();
-
-            try {
-                authService.authenticate();
-            } catch (VripperException e) {
-                logger.error("Cannot authenticate user with ViperGirls", e);
-            }
-        }
-
-        private void registerShutdownHook() {
-            Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
-        }
-    }
-
-    @Component
-    public class TestRunner implements CommandLineRunner {
-
-        @Autowired
-        private PostParser postParser;
-
-        @Override
-        public void run(String... args) throws Exception {
-
-//            long start = System.currentTimeMillis();
-//            PostParser.VRThreadParser vrThreadParser = postParser.new VRThreadParser("600408");
-//            vrThreadParser.getPostPublishProcessor().subscribe(e -> System.out.println("new Post: " + e.getPostId()));
-//            vrThreadParser.parse();
-//            System.out.println("Total time: " + (System.currentTimeMillis() - start));
-        }
-    }
 }
 
-

vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java

@@ -48,7 +48,7 @@ public class Image {
         this.postName = postName;
         this.host = host;
         this.index = index;
-        status = Status.PENDING;
+        status = Status.STOPPED;
         imageStateProcessor = BehaviorProcessor.create();
     }
 
@@ -80,7 +80,7 @@ public class Image {
                 .subscribe();
 
         current.set(0);
-        status = Status.PENDING;
+        status = Status.STOPPED;
         imageStateProcessor.onNext(this);
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java

@@ -26,12 +26,12 @@ public class Post {
 
     private String postId;
 
+    private String threadTitle;
+
     private String threadId;
 
     private String title;
 
-    private String postCounter;
-
     private String url;
 
     private List<Image> images;
@@ -46,27 +46,20 @@ public class Post {
 
     private boolean removed = false;
 
-    public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId) {
+    private String forum;
+
+    public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) {
         this.title = title;
         this.url = url;
         this.images = images;
         this.metadata = metadata;
         this.postId = postId;
+        this.forum = forum;
         this.threadId = threadId;
-        if (this.images.contains(null)) {
-            System.out.println("Oops");
-        }
-        this.images.stream().map(e -> {
-            if (e == null) {
-                System.out.println("Am the cause");
-            }
-            return e.getHost();
-        }).collect(Collectors.toSet());
-        this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
-        this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
+        this.threadTitle = threadTitle;
+        this.hosts = this.images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet());
         total = images.size();
-        status = Status.PENDING;
-
+        status = Status.STOPPED;
     }
 
     public void setAppStateService(AppStateService appStateService) {

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/persistance/ImagePersistanceMixin.java

@@ -5,10 +5,14 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.BehaviorProcessor;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class ImagePersistanceMixin {
 
     @JsonSerialize(converter = HostToString.class)

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/persistance/PostPersistanceMixin.java

@@ -1,8 +1,12 @@
 package tn.mnlr.vripper.entities.mixin.persistance;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class PostPersistanceMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/ui/ImageUIMixin.java

@@ -3,10 +3,14 @@ package tn.mnlr.vripper.entities.mixin.ui;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.BehaviorProcessor;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class ImageUIMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/ui/PostUIMixin.java

@@ -1,11 +1,15 @@
 package tn.mnlr.vripper.entities.mixin.ui;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.List;
 
+@Getter
+@Setter
 public abstract class PostUIMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/exception/DownloadException.java

@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
         super(message);
     }
 
-    public DownloadException(Exception e) {
+    public DownloadException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class HostException extends Exception {
-    public HostException(Exception e) {
+    public HostException(Throwable e) {
         super(e);
     }
 
@@ -9,7 +9,7 @@ public class HostException extends Exception {
         super(message);
     }
 
-    public HostException(String message, Exception e) {
+    public HostException(String message, Throwable e) {
         super(message,e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HtmlProcessorException.java

@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
 
 public class HtmlProcessorException extends Exception {
 
-    public HtmlProcessorException(Exception e) {
+    public HtmlProcessorException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/PostParseException.java

@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
         super(message);
     }
 
-    public PostParseException(String message, Exception e) {
+    public PostParseException(String message, Throwable e) {
         super(message, e);
     }
 
-    public PostParseException(Exception e) {
+    public PostParseException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/VripperException.java

@@ -5,7 +5,7 @@ public class VripperException extends Exception {
         super(message);
     }
 
-    public VripperException(Exception e) {
+    public VripperException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/XpathException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class XpathException extends Exception {
-    public XpathException(Exception e) {
+    public XpathException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java

@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -28,7 +29,7 @@ public class AcidimgHost extends Host {
 
     private static final String host = "acidimg.cc";
     private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
-    public static final String IMG_XPATH = "//img[@class='centred']";
+    private static final String IMG_XPATH = "//img[@class='centred']";
 
     @Autowired
     private ConnectionManager cm;
@@ -39,20 +40,25 @@ public class AcidimgHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node contDiv;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
-            logger.info(String.format("Click button found for %s", url));
+            logger.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpPost httpPost = cm.buildHttpPost(url);
             httpPost.addHeader("Referer", url);
@@ -64,9 +70,9 @@ public class AcidimgHost extends Host {
                 throw new HostException(e);
             }
 
-            logger.info(String.format("Requesting %s", httpPost));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
-                logger.info(String.format("Cleaning response for %s", httpPost));
+            logger.debug(String.format("Requesting %s", httpPost));
+            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
+                logger.debug(String.format("Cleaning response for %s", httpPost));
                 doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
                 EntityUtils.consumeQuietly(response.getEntity());
             } catch (Exception e) {
@@ -76,7 +82,7 @@ public class AcidimgHost extends Host {
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
@@ -87,7 +93,7 @@ public class AcidimgHost extends Host {
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -5,6 +5,8 @@ import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,9 +27,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.Iterator;
-import java.util.Random;
+import java.util.Objects;
 
 @Service
 abstract public class Host {
@@ -45,6 +50,9 @@ abstract public class Host {
     @Autowired
     private AppSettingsService appSettingsService;
 
+    @Autowired
+    private AppStateService appStateService;
+
     @Autowired
     private ConnectionManager cm;
 
@@ -56,57 +64,67 @@ abstract public class Host {
 
     abstract public String getHost();
 
+    abstract public String getLookup();
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(pathService);
+    }
+
     public boolean isSupported(String url) {
-        return url.contains(getHost());
+        return url.contains(getLookup());
     }
 
     public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
 
+        HttpClientContext context = HttpClientContext.create();
+        context.setCookieStore(new BasicCookieStore());
         try {
 
+            appStateService.postDownloadingUpdate(image.getPostId());
+
             imageFileData.setPageUrl(image.getUrl());
 
-            /**
+            /*
              * HOST SPECIFIC
              */
-            logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
-            setNameAndUrl(image.getUrl(), imageFileData);
-            logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
-            logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
+            logger.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
+            setNameAndUrl(image.getUrl(), imageFileData, context);
+            logger.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
+            logger.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
 
-            logger.info(String.format("Building image request for %s", image.getUrl()));
+            logger.debug(String.format("Building image request for %s", image.getUrl()));
             setImageRequest(imageFileData);
-            /**
+            /*
              * END HOST SPECIFIC
              */
 
             String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
-            logger.info(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
+            logger.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
             imageFileData.setImageName(formatImageFileName);
             File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
-            logger.info(String.format("Saving to %s", destinationFolder.getPath()));
+            logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
+            logger.debug(String.format("Creating %s", destinationFolder.getPath()));
+            destinationFolder.mkdirs();
             if (!destinationFolder.exists()) {
-                logger.info(String.format("Creating %s", destinationFolder.getPath()));
-                destinationFolder.mkdirs();
+                logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
             }
 
             HttpClient client = cm.getClient().build();
 
-            logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
+            logger.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
+            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest(), context)) {
 
-                if(response.getStatusLine().getStatusCode() / 100 != 2) {
+                if (response.getStatusLine().getStatusCode() / 100 != 2) {
                     EntityUtils.consumeQuietly(response.getEntity());
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
-                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
-                InputStream downloadStream = response.getEntity().getContent();
-                FileOutputStream fos = new FileOutputStream(outputFile);
-                try {
+                File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
+                try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
                     image.setTotal(response.getEntity().getContentLength());
-                    logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
-                    logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
+                    logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
+                    logger.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
 
                     byte[] buffer = new byte[READ_BUFFER_SIZE];
                     int read;
@@ -115,77 +133,71 @@ abstract public class Host {
                         image.increase(read);
                         downloadSpeedService.increase(read);
                     }
+                    fos.flush();
                     EntityUtils.consumeQuietly(response.getEntity());
-                } finally {
-                    if (downloadStream != null) {
-                        downloadStream.close();
-                    }
-                    if (fos != null) {
-                        fos.close();
-                    }
                 }
-                checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
+                File finalName = checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
+                imageFileData.setFileName(finalName.getName());
             }
         } catch (Exception e) {
-            if(Thread.interrupted()) {
+            if (Thread.interrupted()) {
                 throw new InterruptedException("Download was interrupted");
             }
             throw new DownloadException(e);
         }
     }
 
-    private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
-        String formatName;
+    private File checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
         try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
             Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
             if (!it.hasNext()) {
                 throw new HostException("Image file is not recognized!");
             }
             ImageReader reader = it.next();
-            formatName = reader.getFormatName();
-            if (formatName.toUpperCase().equals("JPEG")) {
-                formatName = "jpg";
+            if (reader.getFormatName().toUpperCase().equals("JPEG")) {
+                String imageNameLC = imageName.toLowerCase();
+                if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
+                    imageName += ".jpg";
+                } else {
+                    String toReplace = null;
+                    if (imageNameLC.endsWith("_jpg")) {
+                        toReplace = "_jpg";
+                    } else if (imageNameLC.endsWith("_jpeg")) {
+                        toReplace = "_jpeg";
+                    }
+                    if (toReplace != null) {
+                        imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
+                    }
+                }
             }
         } catch (Exception e) {
             throw new HostException("Failed to guess image format", e);
         }
         try {
-            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
-            if (outImage.exists()) {
-                outImage.delete();
+            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName);
+            if (outImage.exists() && outImage.delete()) {
+                logger.debug(String.format("%s is deleted", outImage.toString()));
             }
-            Files.move(outputFile.toPath(), outImage.toPath());
+            return Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
         } catch (Exception e) {
             throw new HostException("Failed to rename the image", e);
         }
     }
 
-    /**
-     * Just for testing, you may ignore
-     * @throws Exception
-     */
-    private void randomFail() throws Exception {
-        Random random = new Random();
-        int i = random.nextInt(100);
-        if (i < 50) {
-            throw new Exception("Purpose error");
-        }
-    }
-
-    protected final String getDefaultImageName(final String imgUrl) {
-        String imageTitle  = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
+    final String getDefaultImageName(final String imgUrl) {
+        String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
         logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
         return imgUrl;
     }
 
-    protected final Response getResponse(final String url) throws HostException {
+    final Response getResponse(final String url, final HttpClientContext context) throws HostException {
         String basePage;
 
         HttpClient client = cm.getClient().build();
         HttpGet httpGet = cm.buildHttpGet(url);
         Header[] headers;
-        logger.info(String.format("Requesting %s", url));
-        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
+        logger.debug(String.format("Requesting %s", url));
+        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
             if (response.getStatusLine().getStatusCode() / 100 != 2) {
                 throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
             }
@@ -198,27 +210,51 @@ abstract public class Host {
         }
 
         try {
-            logger.info(String.format("Cleaning %s response", url));
+            logger.debug(String.format("Cleaning %s response", url));
             return new Response(htmlProcessorService.clean(basePage), headers);
         } catch (HtmlProcessorException e) {
             throw new HostException(e);
         }
     }
 
-    protected void setImageRequest(final ImageFileData imageFileData) {
+    protected String appendUri(String uri, String appendQuery) throws URISyntaxException {
+        URI oldUri = new URI(uri);
+
+        String newQuery = oldUri.getQuery();
+        if (newQuery == null) {
+            newQuery = appendQuery;
+        } else {
+            newQuery += "&" + appendQuery;
+        }
+
+        return new URI(oldUri.getScheme(), oldUri.getAuthority(),
+                oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
+    }
+
+    private void setImageRequest(final ImageFileData imageFileData) {
         HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
         httpGet.addHeader("Referer", imageFileData.getPageUrl());
         imageFileData.setImageRequest(httpGet);
     }
 
-    protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
+    protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Host host = (Host) o;
+        return Objects.equals(getHost(), host.getHost());
+    }
 
     @Getter
     public static class Response {
+
         protected Response(Document document, Header[] headers) {
             this.document = document;
             this.headers = headers;
         }
+
         private Document document;
         private Header[] headers;
     }

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageBamHost.java

@@ -1,9 +1,9 @@
 package tn.mnlr.vripper.host;
 
-import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -18,8 +18,6 @@ import tn.mnlr.vripper.q.ImageFileData;
 import tn.mnlr.vripper.services.ConnectionManager;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.stream.Collectors;
 
 @Service
 public class ImageBamHost extends Host {
@@ -27,8 +25,8 @@ public class ImageBamHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
 
     private static final String host = "imagebam.com";
-    public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
-    public static final String IMG_XPATH = "//img[@class='image']";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@class='image']";
 
     @Autowired
     private ConnectionManager cm;
@@ -39,34 +37,31 @@ public class ImageBamHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Response response = getResponse(url);
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Response response = getResponse(url, context);
         Document doc = response.getDocument();
-        Header[] headers = response.getHeaders();
-        String cookies = Arrays.asList(headers)
-                .stream()
-                .filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
-                .map(e -> e.getValue().split(";")[0].trim())
-                .collect(Collectors.joining("; "));
 
         Node contDiv;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
-            logger.info(String.format("Getting cookies to use for %s", url));
-            logger.info(String.format("Click button found for %s", url));
+            logger.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpGet httpGet = cm.buildHttpGet(url);
             httpGet.addHeader("Referer", url);
-            httpGet.addHeader("Cookie", cookies);
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
+            logger.debug(String.format("Requesting %s", httpGet));
+            try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
                 String s = EntityUtils.toString(res.getEntity());
                 logger.debug(String.format("%s response is:%n%s", httpGet, s));
                 logger.debug(String.format("Cleaning response for %s", httpGet));
@@ -79,14 +74,14 @@ public class ImageBamHost extends Host {
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageTwistHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -16,7 +17,7 @@ import java.util.Optional;
 @Service
 public class ImageTwistHost extends Host {
 
-    public static final String IMG_XPATH = "//img[contains(@class, 'img')]";
+    private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
     private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
     private static final String host = "imagetwist.com";
     @Autowired
@@ -28,20 +29,25 @@ public class ImageTwistHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageVenueHost.java

@@ -0,0 +1,79 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.net.URI;
+
+@Service
+public class ImageVenueHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImageVenueHost.class);
+
+    private static final String host = "imagevenue.com";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@id='thepic']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        //Sadly, they do not support https.
+        //If they add such support in the future,
+        //then we should automatically adapt the URL here as done elsewhere.
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            if(xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
+                //Button detected. No need to actually click it, just make the call again.
+                resp = getResponse(url, context);
+                doc = resp.getDocument();
+            }
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        if (imgNode == null) {
+            throw new HostException("Failed to locate image");
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            URI baseUri = new URI(url);
+            imageFileData.setImageUrl(new URI(baseUri.getScheme(), baseUri.getHost(), '/'+imgUrl, null).toString());
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch(Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +18,8 @@ public class ImageZillaHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
 
     private static final String host = "imagezilla.net";
-    public static final String IMG_XPATH = "//img[@id='photo']";
+    private static final String lookup = "imagezilla.net/show";
+    private static final String IMG_XPATH = "//img[@id='photo']";
 
     @Autowired
     private ConnectionManager cm;
@@ -28,15 +30,20 @@ public class ImageZillaHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return lookup;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         String title;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
-            logger.info(String.format("Resolving name for %s", url));
+            logger.debug(String.format("Resolving name for %s", url));
             if(titleNode != null) {
                 title = titleNode.getTextContent().trim();
             } else {

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgSpiceHost.java

@@ -0,0 +1,62 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class ImgSpiceHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImgSpiceHost.class);
+
+    private static final String host = "imgspice.com";
+    private static final String IMG_XPATH = "//img[@id='imgpreview']";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +18,7 @@ public class ImgboxHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
 
     private static final String host = "imgbox.com";
-    public static final String IMG_XPATH = "//img[@id='img']";
+    private static final String IMG_XPATH = "//img[@id='img']";
 
     @Autowired
     private ConnectionManager cm;
@@ -28,20 +29,25 @@ public class ImgboxHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -29,8 +30,8 @@ public class ImxHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
 
     private static final String host = "imx.to";
-    public static final String CONTINUE_BUTTON_XPATH = "//div[@id='continuetoimage']";
-    public static final String IMG_XPATH = "//img[@class='centred']";
+    private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
+    private static final String IMG_XPATH = "//img[@class='centred']";
 
     @Autowired
     private ConnectionManager cm;
@@ -41,50 +42,62 @@ public class ImxHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
 
         Node contDiv;
+        String value = null;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
+            Node node = contDiv.getAttributes().getNamedItem("value");
+            if (node != null) {
+                value = node.getTextContent();
+            }
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
-        if (contDiv != null) {
-            logger.info(String.format("Click button found for %s", url));
-            HttpClient client = cm.getClient().build();
-            HttpPost httpPost = cm.buildHttpPost(url);
-            httpPost.addHeader("Referer", url);
-            List<NameValuePair> params = new ArrayList<>();
-            params.add(new BasicNameValuePair("imgContinue", "Continue to image ... "));
-            try {
-                httpPost.setEntity(new UrlEncodedFormEntity(params));
-            } catch (Exception e) {
-                throw new HostException(e);
-            }
-            logger.info(String.format("Requesting %s", httpPost));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
-                logger.debug(String.format("Cleaning response for %s", httpPost));
-                doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (IOException | HtmlProcessorException e) {
-                throw new HostException(e);
-            }
+        if (value == null) {
+            throw new HostException("Failed to obtain value attribute from continue input");
+        }
+        logger.debug(String.format("Click button found for %s", url));
+        HttpClient client = cm.getClient().build();
+        HttpPost httpPost = cm.buildHttpPost(url);
+        List<NameValuePair> params = new ArrayList<>();
+        params.add(new BasicNameValuePair("imgContinue", value));
+        try {
+            httpPost.setEntity(new UrlEncodedFormEntity(params));
+        } catch (Exception e) {
+            throw new HostException(e);
+        }
+        logger.debug(String.format("Requesting %s", httpPost));
+        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
+            logger.debug(String.format("Cleaning response for %s", httpPost));
+            doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
+            EntityUtils.consumeQuietly(response.getEntity());
+        } catch (IOException | HtmlProcessorException e) {
+            throw new HostException(e);
         }
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/PimpandhostHost.java

@@ -0,0 +1,68 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class PimpandhostHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PimpandhostHost.class);
+
+    private static final String host = "pimpandhost.com";
+    private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url;
+        try {
+            url = appendUri(_url.replace("http://", "https://"), "size=original");
+        } catch (Exception e) {
+            throw new HostException(e);
+        }
+
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
+            String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +18,8 @@ public class PixhostHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
 
     private static final String host = "pixhost.to";
-    public static final String IMG_XPATH = "//img[@id='image']";
+    private static final String lookup = "pixhost.to/show";
+    private static final String IMG_XPATH = "//img[@id='image']";
 
     @Autowired
     private ConnectionManager cm;
@@ -28,20 +30,25 @@ public class PixhostHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return lookup;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/PixxxelsHost.java

@@ -0,0 +1,67 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+@Service
+public class PixxxelsHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PixxxelsHost.class);
+
+    private static final String host = "pixxxels.cc";
+    private static final String IMG_XPATH = "//*[@id='download']";
+    private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
+
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
+
+        Node imgNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = titleNode.getTextContent().trim();
+            String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/PostImgHost.java

@@ -0,0 +1,61 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+
+import java.util.Optional;
+
+@Service
+public class PostImgHost extends Host {
+
+    private static final Logger logger = LoggerFactory.getLogger(PostImgHost.class);
+
+    private static final String host = "postimg.cc";
+    private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
+    private static final String IMG_XPATH = "//a[@id='download']";
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Document doc = getResponse(url, context).getDocument();
+
+        Node urlNode, titleNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
+
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            urlNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> getDefaultImageName(url));
+
+            imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
+            imageFileData.setImageName(imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/TurboImageHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,8 +18,8 @@ public class TurboImageHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
 
     private static final String host = "turboimagehost.com";
-    public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
-    public static final String IMG_XPATH = "//img[@id='uImage']";
+    private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
+    private static final String IMG_XPATH = "//img[@id='uImage']";
 
     @Autowired
     private ConnectionManager cm;
@@ -29,15 +30,20 @@ public class TurboImageHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         String title;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
             Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
-            logger.info(String.format("Resolving name for %s", url));
+            logger.debug(String.format("Resolving name for %s", url));
             if(titleNode != null) {
                 title = titleNode.getTextContent().trim();
             } else {

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java

@@ -18,14 +18,14 @@ public class DownloadJob implements Callable<Image> {
     @Getter
     private final ImageFileData imageFileData = new ImageFileData();
 
-    public DownloadJob(Image image) {
+    DownloadJob(Image image) {
         this.image = image;
     }
 
     @Override
     public Image call() throws Exception {
 
-        logger.info(String.format("Starting downloading %s", image.getUrl()));
+        logger.debug(String.format("Starting downloading %s", image.getUrl()));
         image.setStatus(Image.Status.DOWNLOADING);
         image.setCurrent(0);
         image.getHost().download(image, imageFileData);

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java

@@ -1,51 +1,70 @@
 package tn.mnlr.vripper.q;
 
-import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
 
-import java.util.Arrays;
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
+import java.util.Map;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingDeque;
 
 @Service
 public class DownloadQ {
 
-    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
-
     private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
 
     @Autowired
     private AppStateService appStateService;
 
+    private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> downloadQ = new ConcurrentHashMap<>();
     @Autowired
-    private ExecutionService executionService;
+    private AppSettingsService appSettingsService;
+    @Autowired
+    private List<Host> hosts;
 
-    private BlockingQueue<DownloadJob> downloadQ = new LinkedBlockingQueue<>();
+    @PostConstruct
+    private void init() {
+        hosts.forEach(host -> downloadQ.put(host, new LinkedBlockingDeque<>()));
+    }
 
-    @Getter
-    private boolean notPauseQ = true;
-
-    public synchronized void put(Image image) throws InterruptedException {
-        logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
+    public void put(Image image) throws InterruptedException {
+        logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
         image.init();
         DownloadJob downloadJob = new DownloadJob(image);
-        downloadQ.put(downloadJob);
+        downloadQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
         appStateService.newDownloadJob(downloadJob);
     }
 
-    public DownloadJob take() throws InterruptedException {
-        DownloadJob downloadJob = downloadQ.take();
-        logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
-        return downloadJob;
+    public void remove(final DownloadJob downloadJob) {
+        downloadQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
+    }
+
+    public List<DownloadJob> peek() {
+        List<DownloadJob> downloadJobs = new ArrayList<>();
+        if (hosts.size() == 0) {
+            return downloadJobs;
+        }
+        for (Host host : hosts) {
+            Iterator<DownloadJob> it = downloadQ.get(host).iterator();
+            for (int i = 0; i < appSettingsService.getMaxThreads(); i++) {
+                DownloadJob downloadJob = it.hasNext() ? it.next() : null;
+                if (downloadJob != null) {
+                    downloadJobs.add(downloadJob);
+                }
+            }
+        }
+        return downloadJobs;
     }
 
     public void enqueue(Post post) throws InterruptedException {
@@ -54,91 +73,11 @@ public class DownloadQ {
         }
     }
 
-    public void restart(String postId) throws InterruptedException {
-        if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
-            logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
-            return;
-        }
-        List<Image> images = appStateService.getPost(postId)
-                .getImages()
-                .stream()
-                .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                .collect(Collectors.toList());
-        if (images.isEmpty()) {
-            return;
-        }
-        appStateService.getPost(postId).setStatus(Post.Status.PENDING);
-        logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
-        for (Image image : images) {
-            put(image);
-        }
-    }
-
-    public void removeScheduled(Image image) {
-        image.setStatus(Image.Status.STOPPED);
-        logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
-
-        Iterator<DownloadJob> iterator = downloadQ.iterator();
-        boolean removed = false;
-        while(iterator.hasNext()) {
-            DownloadJob next = iterator.next();
-            if(next.getImage().getPostId().equals(image.getPostId())) {
-                iterator.remove();
-                appStateService.doneDownloadJob(image);
-                logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
-                removed = true;
-                break;
-            }
-        }
-
-        if(!removed) {
-            logger.warn(String.format("Job for %s does not exist", image.getUrl()));
-        }
-
-        image.cleanup();
-    }
-
-    public void removeRunning(String postId) {
-        logger.info(String.format("Interrupting running jobs for post id %s", postId));
-        executionService.stop(postId);
-    }
-
-
-    public synchronized void stop(String postId) {
-        try {
-            if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
-                return;
-            }
-            notPauseQ = false;
-            appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
-            List<Image> images = appStateService.getPost(postId)
-                    .getImages()
-                    .stream()
-                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                    .collect(Collectors.toList());
-            if (images.isEmpty()) {
-                return;
-            }
-            logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
-            images.forEach(image -> removeScheduled(image));
-            removeRunning(postId);
-        } finally {
-            notPauseQ = true;
-        }
-    }
-
     public int size() {
-        return downloadQ.size();
+        return downloadQ.values().stream().mapToInt(BlockingDeque::size).sum();
     }
 
-    public synchronized void stopAll() {
-        appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
-    }
-
-    public synchronized void restartAll() throws InterruptedException {
-        for (Post post : appStateService.getCurrentPosts().values()) {
-            String postId = post.getPostId();
-            restart(postId);
-        }
+    public Iterable<? extends Map.Entry<Host, BlockingDeque<DownloadJob>>> entries() {
+        return downloadQ.entrySet();
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -7,17 +7,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
+import tn.mnlr.vripper.services.ThumbnailGenerator;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.time.temporal.ChronoUnit;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -27,6 +29,8 @@ public class ExecutionService {
 
     private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
 
+    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
+
     @Autowired
     private DownloadQ downloadQ;
 
@@ -36,29 +40,40 @@ public class ExecutionService {
     @Autowired
     private AppStateService appStateService;
 
-    private AtomicInteger threadCount = new AtomicInteger();
+    @Autowired
+    private ThumbnailGenerator thumbnailGenerator;
 
-    private ExecutorService executor = Executors.newFixedThreadPool(10);
+    @Autowired
+    private AppSettingsService appSettingsService;
+    private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
+    private boolean notPauseQ = true;
+    private ExecutorService executor = Executors.newFixedThreadPool(20);
 
     private Thread executionThread;
 
+    BlockingQueue<DownloadJob> queue = new LinkedBlockingQueue<>();
+
     private RetryPolicy<Object> retryPolicy;
 
     private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
 
     private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
+    private Thread pollThread;
 
     @PostConstruct
     private void init() {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withDelay(Duration.ofSeconds(5))
-                .withMaxRetries(2)
+                .withBackoff(1, 10, ChronoUnit.SECONDS)
+                .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+                .withMaxAttempts(3)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 
         executionThread = new Thread(this::start, "Executor thread");
+        pollThread = new Thread(this::poll, "Polling thread");
+        pollThread.start();
         executionThread.start();
     }
 
@@ -68,92 +83,198 @@ public class ExecutionService {
         executionThread.interrupt();
         executor.shutdown();
         appStateService.getCurrentPosts().keySet().forEach(p -> {
-            logger.info(String.format("Stopping download jobs for %s", p));
-            this.stop(p);
+            logger.debug(String.format("Stopping download jobs for %s", p));
+            this.stopRunning(p);
         });
         executor.awaitTermination(10, TimeUnit.SECONDS);
     }
 
-    public void stop(String postId) {
+    public void stopRunning(String postId) {
         List<DownloadJob> data = running
                 .stream()
                 .filter(e -> e.getImage().getPostId().equals(postId))
                 .peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
                 .collect(Collectors.toList());
-        logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
+        logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
 
         data.forEach(e -> {
             futures.get(e.getImage().getUrl()).cancel(true);
-            if(e.getImageFileData().getImageRequest() != null) {
+            if (e.getImageFileData().getImageRequest() != null) {
                 e.getImageFileData().getImageRequest().abort();
             }
             e.getImage().cleanup();
         });
     }
 
-    boolean canRun() {
-        boolean canRun = threadCount.get() < settings.getMaxThreads();
-        if (canRun && downloadQ.isNotPauseQ()) {
-            threadCount.incrementAndGet();
+
+    public synchronized void stopAll() {
+        appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
+    }
+
+    public synchronized void restartAll() throws InterruptedException {
+        for (Post post : appStateService.getCurrentPosts().values()) {
+            String postId = post.getPostId();
+            restart(postId);
+        }
+    }
+
+    public synchronized void restart(String postId) throws InterruptedException {
+        if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
+            logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
+            return;
+        }
+        List<Image> images = appStateService.getPost(postId)
+                .getImages()
+                .stream()
+                .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                .collect(Collectors.toList());
+        if (images.isEmpty()) {
+            return;
+        }
+        appStateService.getPost(postId).setStatus(Post.Status.PENDING);
+        logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
+        for (Image image : images) {
+            downloadQ.put(image);
+        }
+    }
+
+    private synchronized void removeScheduled(Image image) {
+        image.setStatus(Image.Status.STOPPED);
+        logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
+
+        boolean removed = false;
+        main:
+        for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : downloadQ.entries()) {
+            Iterator<DownloadJob> iterator = entry.getValue().iterator();
+            while (iterator.hasNext()) {
+                DownloadJob next = iterator.next();
+                if (next.getImage().getPostId().equals(image.getPostId())) {
+                    iterator.remove();
+                    appStateService.doneDownloadJob(image);
+                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
+                    removed = true;
+                    break main;
+                }
+            }
+        }
+
+        if (!removed) {
+            logger.debug(String.format("Job for %s does not exist", image.getUrl()));
+        }
+
+        image.cleanup();
+    }
+
+    private synchronized void removeRunning(String postId) {
+        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
+        stopRunning(postId);
+    }
+
+    public synchronized void stop(String postId) {
+        try {
+            if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
+                return;
+            }
+            notPauseQ = false;
+            appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
+            List<Image> images = appStateService.getPost(postId)
+                    .getImages()
+                    .stream()
+                    .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                    .collect(Collectors.toList());
+            if (images.isEmpty()) {
+                return;
+            }
+            logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
+            images.forEach(this::removeScheduled);
+            removeRunning(postId);
+        } finally {
+            notPauseQ = true;
+        }
+    }
+
+    private boolean canRun(Host host) {
+        boolean canRun;
+        AtomicInteger count = threadCount.get(host);
+        if (count == null) {
+            threadCount.put(host, new AtomicInteger(0));
+        }
+        canRun = threadCount.get(host).get() < settings.getMaxThreads() && (settings.getMaxTotalThreads() == 0 || threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getMaxTotalThreads());
+        if (canRun && notPauseQ) {
+            threadCount.get(host).incrementAndGet();
             return true;
         }
         return false;
     }
 
-    public void start() {
+    public void poll() {
         while (!Thread.interrupted()) {
-            if (canRun()) {
-                DownloadJob take = null;
-                try {
-                    take = downloadQ.take();
-                    if (take == null) {
-                        continue;
+            try {
+                List<DownloadJob> peek = downloadQ.peek();
+                for (DownloadJob downloadJob : peek) {
+                    if (canRun(downloadJob.getImage().getHost())) {
+                        queue.offer(downloadJob);
+                        downloadQ.remove(downloadJob);
                     }
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    break;
                 }
-
-                DownloadJob finalTake = take;
-                Runnable task = () -> {
-                    running.add(finalTake);
-
-                    Failsafe.with(retryPolicy)
-                            .onFailure(e -> {
-                                if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
-                                    logger.info("Job successfully interrupted");
-                                    return;
-                                }
-                                logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
-                                finalTake.getImage().setStatus(Image.Status.ERROR);
-                            })
-                            .onComplete(e -> {
-                                appStateService.doneDownloadJob(finalTake.getImage());
-                                logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
-                                synchronized (threadCount) {
-                                    threadCount.decrementAndGet();
-                                    running.remove(finalTake);
-                                    futures.remove(finalTake.getImage().getUrl());
-                                    threadCount.notify();
-                                }
-                            })
-                            .get(finalTake::call);
-                };
-                logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
-                futures.put(finalTake.getImage().getUrl(), executor.submit(task));
-            } else {
                 synchronized (threadCount) {
-                    try {
-                        threadCount.wait(2_000);
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                        break;
-                    }
+                    threadCount.wait(2_000);
                 }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
             }
         }
     }
 
+    public void start() {
+        while (!Thread.interrupted()) {
+            try {
+                push(queue.take());
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            } catch (Exception e) {
+                logger.error("Execution Service failed", e);
+                break;
+            }
+        }
+    }
+
+    private void push(DownloadJob take) {
+        Runnable task = () -> {
+            running.add(take);
+
+            Failsafe.with(retryPolicy)
+                    .onFailure(e -> {
+                        if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
+                            logger.debug("Job successfully interrupted");
+                            return;
+                        }
+                        logger.error(String.format("Failed to download %s after %d tries", take.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
+                        take.getImage().setStatus(Image.Status.ERROR);
+                    })
+                    .onComplete(e -> {
+                        appStateService.doneDownloadJob(take.getImage());
+                        logger.debug(String.format("Finished downloading %s", take.getImage().getUrl()));
+                        if (appSettingsService.isViewPhotos()) {
+                            VripperApplication.commonExecutor.submit(
+                                    () -> thumbnailGenerator.getThumbnails()
+                                            .get(new ThumbnailGenerator.CacheKey(take.getImage().getPostId(), take.getImageFileData().getFileName())));
+                        }
+                        threadCount.get(take.getImage().getHost()).decrementAndGet();
+                        running.remove(take);
+                        futures.remove(take.getImage().getUrl());
+                        synchronized (threadCount) {
+                            threadCount.notify();
+                        }
+                    })
+                    .get(take::call);
+        };
+        logger.debug(String.format("Scheduling a job for %s", take.getImage().getUrl()));
+        futures.put(take.getImage().getUrl(), executor.submit(task));
+    }
+
     public int runningCount() {
         return running.size();
     }

vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java

@@ -10,6 +10,7 @@ public class ImageFileData {
 
     private String pageUrl;
     private String imageName;
+    private String fileName;
     private String imageUrl;
     private HttpUriRequest imageRequest;
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -2,13 +2,16 @@ package tn.mnlr.vripper.services;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.exception.ValidationException;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
@@ -22,12 +25,18 @@ import java.util.prefs.Preferences;
 @Setter
 public class AppSettingsService {
 
+    private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
+
+    @Value("${base.dir}")
+    private String defaultDownloadPath;
+
     private static final Logger logger = LoggerFactory.getLogger(AppSettingsService.class);
 
     private Preferences prefs = Preferences.userNodeForPackage(AppSettingsService.class);
 
     private final String DOWNLOAD_PATH = "DOWNLOAD_PATH";
     private final String MAX_THREADS = "MAX_THREADS";
+    private int maxTotalThreads;
     private final String AUTO_START = "AUTO_START";
     private final String V_LOGIN = "VLOGIN";
     private final String V_USERNAME = "VUSERNAME";
@@ -35,6 +44,17 @@ public class AppSettingsService {
     private final String V_THANKS = "VTHANKS";
     private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
     private final String FORCE_ORDER = "FORCE_ORDER";
+    private final String SUBFOLDER = "SUBFOLDER";
+    private final String THREADSUBFOLDER = "THREADSUBFOLDER";
+    private final String CLEAR = "CLEAR";
+    private final String DARK_THEME = "DARK_THEME";
+    private final String VIEW_PHOTOS = "VIEW_PHOTOS";
+    private final String NOTIFICATION = "NOTIFICATION";
+
+    @PostConstruct
+    private void init() {
+        restore();
+    }
 
     private String downloadPath;
     private int maxThreads;
@@ -44,7 +64,13 @@ public class AppSettingsService {
     private String vPassword;
     private boolean vThanks;
     private boolean desktopClipboard;
+    private boolean subLocation;
+    private boolean threadSubLocation;
     private boolean forceOrder;
+    private boolean clearCompleted;
+    private boolean darkTheme;
+    private boolean viewPhotos;
+    private boolean notificationEnabled;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -56,8 +82,9 @@ public class AppSettingsService {
 
     public void restore() {
 
-        downloadPath = prefs.get(DOWNLOAD_PATH, System.getProperty("user.dir"));
+        downloadPath = prefs.get(DOWNLOAD_PATH, defaultDownloadPath);
         maxThreads = prefs.getInt(MAX_THREADS, 4);
+        maxTotalThreads = prefs.getInt(MAX_TOTAL_THREADS, 0);
         autoStart = prefs.getBoolean(AUTO_START, true);
         vLogin = prefs.getBoolean(V_LOGIN, false);
         vUsername = prefs.get(V_USERNAME, "");
@@ -65,6 +92,12 @@ public class AppSettingsService {
         vThanks = prefs.getBoolean(V_THANKS, false);
         desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
         forceOrder = prefs.getBoolean(FORCE_ORDER, false);
+        subLocation = prefs.getBoolean(SUBFOLDER, false);
+        threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
+        clearCompleted = prefs.getBoolean(CLEAR, false);
+        darkTheme = prefs.getBoolean(DARK_THEME, false);
+        viewPhotos = prefs.getBoolean(VIEW_PHOTOS, false);
+        notificationEnabled = prefs.getBoolean(NOTIFICATION, false);
     }
 
     @PreDestroy
@@ -72,6 +105,7 @@ public class AppSettingsService {
 
         prefs.put(DOWNLOAD_PATH, downloadPath);
         prefs.putInt(MAX_THREADS, maxThreads);
+        prefs.putInt(MAX_TOTAL_THREADS, maxTotalThreads);
         prefs.putBoolean(AUTO_START, autoStart);
         prefs.putBoolean(V_LOGIN, vLogin);
         prefs.put(V_USERNAME, vUsername);
@@ -79,6 +113,12 @@ public class AppSettingsService {
         prefs.putBoolean(V_THANKS, vThanks);
         prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
         prefs.putBoolean(FORCE_ORDER, forceOrder);
+        prefs.putBoolean(SUBFOLDER, subLocation);
+        prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
+        prefs.putBoolean(CLEAR, clearCompleted);
+        prefs.putBoolean(DARK_THEME, darkTheme);
+        prefs.putBoolean(VIEW_PHOTOS, viewPhotos);
+        prefs.putBoolean(NOTIFICATION, notificationEnabled);
 
         try {
             prefs.sync();
@@ -101,8 +141,33 @@ public class AppSettingsService {
             throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
         }
 
-        if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 8) {
-            throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 8));
+        if (settings.getMaxTotalThreads() < 0) {
+            throw new ValidationException(String.format("Invalid max global concurrent download settings, values must be in greater than %d", 1));
+        }
+
+        if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
+            throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
+        }
+    }
+
+    public Theme getTheme() {
+        return new Theme(this.darkTheme);
+    }
+
+    public void setTheme(Theme theme) {
+        this.darkTheme = theme.darkTheme;
+        save();
+    }
+
+    @Getter
+    @NoArgsConstructor
+    public static class Theme {
+
+        @JsonProperty("darkTheme")
+        private boolean darkTheme;
+
+        Theme(boolean darkTheme) {
+            this.darkTheme = darkTheme;
         }
     }
 
@@ -113,6 +178,8 @@ public class AppSettingsService {
         private String downloadPath;
         @JsonProperty("maxThreads")
         private int maxThreads;
+        @JsonProperty("maxTotalThreads")
+        private int maxTotalThreads;
         @JsonProperty("autoStart")
         private boolean autoStart;
         @JsonProperty("vLogin")
@@ -127,10 +194,21 @@ public class AppSettingsService {
         private boolean desktopClipboard;
         @JsonProperty("forceOrder")
         private boolean forceOrder;
+        @JsonProperty("subLocation")
+        private boolean subLocation;
+        @JsonProperty("threadSubLocation")
+        private boolean threadSubLocation;
+        @JsonProperty("clearCompleted")
+        private boolean clearCompleted;
+        @JsonProperty("viewPhotos")
+        private boolean viewPhotos;
+        @JsonProperty("notification")
+        private boolean notification;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder) {
+        public Settings(String downloadPath, int maxThreads, int maxTotalThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted, boolean viewPhotos, boolean notification) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
+            this.maxTotalThreads = maxTotalThreads;
             this.autoStart = autoStart;
             this.vLogin = vLogin;
             this.vUsername = vUsername;
@@ -138,6 +216,11 @@ public class AppSettingsService {
             this.vThanks = vThanks;
             this.desktopClipboard = desktopClipboard;
             this.forceOrder = forceOrder;
+            this.subLocation = subLocation;
+            this.threadSubLocation = threadSubLocation;
+            this.clearCompleted = clearCompleted;
+            this.viewPhotos = viewPhotos;
+            this.notification = notification;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -11,7 +11,6 @@ import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.q.DownloadJob;
 
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -30,6 +29,10 @@ public class AppStateService {
 
     private Map<String, AtomicInteger> runningPosts = new ConcurrentHashMap<>();
 
+    private Map<String, QueuedVGLink> grabQueue = new ConcurrentHashMap<>();
+
+    private PublishProcessor<QueuedVGLink> liveGrabQueue = PublishProcessor.create();
+
     private PublishProcessor<Image> liveImageUpdates = PublishProcessor.create();
 
     private PublishProcessor<Post> livePostsState = PublishProcessor.create();
@@ -37,6 +40,9 @@ public class AppStateService {
     @Autowired
     private PersistenceService persistenceService;
 
+    @Autowired
+    private AppSettingsService appSettingsService;
+
     public void onImageUpdate(Image imageState) {
 
         persistenceService.getProcessor().onNext(currentPosts);
@@ -53,12 +59,15 @@ public class AppStateService {
         return currentPosts.get(postId);
     }
 
-    public synchronized void newDownloadJob(DownloadJob downloadJob) {
+    public void newDownloadJob(DownloadJob downloadJob) {
         String postId = downloadJob.getImage().getPostId();
         checkKeyRunningPosts(postId);
-        int i = runningPosts.get(postId).incrementAndGet();
-        if (i > 0) {
-            Post post = currentPosts.get(postId);
+        runningPosts.get(postId).incrementAndGet();
+    }
+
+    public synchronized void postDownloadingUpdate(String postId) {
+        Post post = currentPosts.get(postId);
+        if (!post.getStatus().equals(Post.Status.DOWNLOADING) && !post.getStatus().equals(Post.Status.PARTIAL)) {
             post.setStatus(Post.Status.DOWNLOADING);
             livePostsState.onNext(post);
         }
@@ -72,14 +81,16 @@ public class AppStateService {
             post.setStatus(Post.Status.PARTIAL);
         }
         if (i == 0) {
-            if (post.getImages().stream().map(Image::getStatus).filter(e -> e.equals(Image.Status.ERROR)).count() > 0) {
+            if (post.getImages().stream().map(Image::getStatus).anyMatch(e -> e.equals(Image.Status.ERROR))) {
                 post.setStatus(Post.Status.ERROR);
             } else {
                 if (!Post.Status.STOPPED.equals(post.getStatus())) {
                     post.setStatus(Post.Status.COMPLETE);
+                    if (appSettingsService.isClearCompleted()) {
+                        remove(image.getPostId());
+                    }
                 }
             }
-            livePostsState.onNext(post);
         }
     }
 
@@ -93,13 +104,7 @@ public class AppStateService {
         currentPosts.get(postId).setRemoved(true);
         runningPosts.remove(postId);
         currentPosts.remove(postId);
-        Iterator<Map.Entry<String, Image>> iterator = currentImages.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, Image> entry = iterator.next();
-            if(entry.getValue().getPostId().equals(postId)) {
-                iterator.remove();
-            }
-        }
+        currentImages.entrySet().removeIf(entry -> entry.getValue().getPostId().equals(postId));
         persistenceService.getProcessor().onNext(currentPosts);
     }
 
@@ -124,6 +129,7 @@ public class AppStateService {
         return toRemove;
     }
 
+
     @Getter
     public static class CachedThread {
 

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -4,15 +4,19 @@ import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
 import java.net.URI;
+import java.util.concurrent.TimeUnit;
 
 @Service
+@EnableScheduling
 public class ConnectionManager {
 
     private ConnectionManager() {
@@ -22,34 +26,41 @@ public class ConnectionManager {
     private PoolingHttpClientConnectionManager pcm;
 
     private RequestConfig rc = RequestConfig.custom()
-            .setConnectionRequestTimeout(10_000)
-            .setConnectTimeout(10_000)
-            .setSocketTimeout(10_000)
+            .setConnectionRequestTimeout(5_000)
+            .setConnectTimeout(5_000)
+            .setSocketTimeout(5_000)
             .setCookieSpec(CookieSpecs.STANDARD)
             .build();
 
     private void buildConnectionPool() {
 
         pcm = new PoolingHttpClientConnectionManager();
-        pcm.setMaxTotal(200);
-        pcm.setDefaultMaxPerRoute(10);
+        pcm.setMaxTotal(50);
+        pcm.setDefaultMaxPerRoute(4);
+    }
+
+    @Scheduled(fixedDelay = 1000)
+    private void idleConnectionMonitoring() {
+        pcm.closeExpiredConnections();
+        pcm.closeIdleConnections(30, TimeUnit.SECONDS);
     }
 
     public HttpClientBuilder getClient() {
         return HttpClients.custom()
                 .setConnectionManager(pcm)
-                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
+                .setRedirectStrategy(new LaxRedirectStrategy())
+                .disableAutomaticRetries()
                 .setDefaultRequestConfig(rc);
     }
 
     public HttpGet buildHttpGet(String url) {
-        HttpGet httpGet = new HttpGet(url);
+        HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
         httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpGet;
     }
 
     public HttpPost buildHttpPost(String url) {
-        HttpPost httpPost = new HttpPost(url);
+        HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
         httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpPost;
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeed.java

@@ -13,7 +13,7 @@ public class DownloadSpeed {
     }
 
     private String formatSI(long bytes) {
-        return humanReadableByteCount(bytes, true);
+        return humanReadableByteCount(bytes, false);
     }
 
     private String humanReadableByteCount(long bytes, boolean si) {

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalState.java

@@ -13,7 +13,7 @@ public class GlobalState {
     private long remaining;
     private long error;
 
-    public GlobalState(long running, long queued, long remaining, long error) {
+    GlobalState(long running, long queued, long remaining, long error) {
         this.running = running;
         this.queued = queued;
         this.remaining = remaining;
@@ -25,11 +25,7 @@ public class GlobalState {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         GlobalState that = (GlobalState) o;
-        return running == that.running &&
-                queued == that.queued &&
-                remaining == that.remaining &&
-                error == that.error &&
-                Objects.equals(type, that.type);
+        return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
     }
 
     @Override

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java

@@ -29,6 +29,9 @@ public class GlobalStateService {
     @Getter
     private PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
 
+    public GlobalStateService() {
+    }
+
     @Scheduled(fixedDelay = 3000)
     private void interval() {
         GlobalState newGlobalState = new GlobalState(

vripper-server/src/main/java/tn/mnlr/vripper/services/HtmlProcessorService.java

@@ -11,10 +11,6 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
 @Service
 public class HtmlProcessorService {
 
-    private HtmlProcessorService() {
-
-    }
-
     public Document clean(String htmlContent) throws HtmlProcessorException {
         try {
             TagNode clean = new HtmlCleaner().clean(htmlContent);

vripper-server/src/main/java/tn/mnlr/vripper/services/PathService.java

@@ -4,6 +4,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.entities.Post;
 
 import java.io.File;
 
@@ -19,8 +20,12 @@ public class PathService {
     private AppSettingsService appSettingsService;
 
     public final File getDownloadDestinationFolder(String postId) {
-        String postTitle = appStateService.getCurrentPosts().get(postId).getTitle();
-        return new File(appSettingsService.getDownloadPath(), sanitize(postTitle + "_" + postId));
+        Post post = appStateService.getCurrentPosts().get(postId);
+        String postTitle = post.getTitle();
+        String threadTitle = post.getThreadTitle();
+        File sourceFolder = appSettingsService.isSubLocation() ? new File(appSettingsService.getDownloadPath(), sanitize(post.getForum())) : new File(appSettingsService.getDownloadPath());
+        sourceFolder = appSettingsService.isThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
+        return new File(sourceFolder, sanitize(postTitle + "_" + postId));
     }
 
     public final String sanitize(final String folderName) {
@@ -32,17 +37,10 @@ public class PathService {
     /**
      * Will sanitize the image name and remove extension
      *
-     * @param imageName
-     * @return
+     * @param imageName path string
+     * @return Sanitized local path string
      */
     public final String formatImageFileName(String imageName) {
-        int extensionIndex = imageName.lastIndexOf('.');
-        String fileName;
-        if (extensionIndex != -1) {
-            fileName = imageName.substring(0, extensionIndex);
-        } else {
-            fileName = imageName;
-        }
-        return sanitize(fileName);
+        return sanitize(imageName);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java

@@ -7,13 +7,16 @@ import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
-import tn.mnlr.vripper.VripperApplication;
+import tn.mnlr.vripper.EventListenerBean;
+import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
 import tn.mnlr.vripper.entities.mixin.persistance.PostPersistanceMixin;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.io.File;
 import java.io.FileWriter;
@@ -23,10 +26,10 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @Service
 public class PersistenceService {
@@ -36,6 +39,12 @@ public class PersistenceService {
     @Autowired
     private AppStateService stateService;
 
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Getter
+    private String dataPath;
+
     private ObjectMapper om;
 
     private Disposable subscription;
@@ -44,45 +53,58 @@ public class PersistenceService {
     private PublishProcessor<Map<String, Post>> processor = PublishProcessor.create();
 
     private PersistenceService() {
-        File dataFile = new File(VripperApplication.dataPath);
-        if(!dataFile.exists()) {
-            try {
-                if(dataFile.createNewFile()) {
-                    logger.info("Data file successfully created");
-                    try (FileWriter fw = new FileWriter(dataFile)) {
-                        fw.write("{}");
-                    }
-                } else {
-                    logger.info("Data file already exists");
-                }
-            } catch (IOException e) {
-                logger.error("Unable to create data file", e);
-                System.exit(-1);
-            }
-        }
         om = new ObjectMapper();
         om.addMixIn(Image.class, ImagePersistanceMixin.class);
         om.addMixIn(Post.class, PostPersistanceMixin.class);
         subscription = processor
                 .onBackpressureLatest()
-                .buffer(10, TimeUnit.SECONDS)
-                .filter(e -> !e.isEmpty())
-                .map(e -> e.get(0))
+                .throttleLatest(1, TimeUnit.SECONDS)
                 .doOnNext(this::persist)
                 .subscribe();
     }
 
+    @PostConstruct
+    public void init() {
+        dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
+        File dataFile = new File(dataPath);
+        if (!dataFile.exists()) {
+            try {
+                if (dataFile.getParentFile().mkdirs()) {
+                    logger.debug(String.format("%s is created", dataFile.getParentFile().toString()));
+                }
+                if (!dataFile.getParentFile().isDirectory() || !dataFile.getParentFile().canWrite()) {
+                    logger.error(String.format("Unable to write in %s", dataFile.getParent()));
+                    SpringContext.close();
+                }
+
+                if (dataFile.createNewFile()) {
+                    logger.debug("Data file successfully created");
+                    try (FileWriter fw = new FileWriter(dataFile)) {
+                        fw.write("{}");
+                    }
+                } else {
+                    logger.warn("Data file already exists");
+                }
+            } catch (IOException e) {
+                logger.error("Unable to create data file", e);
+                SpringContext.close();
+            }
+        }
+    }
+
     @PreDestroy
     public void preDestroy() {
         this.subscription.dispose();
         logger.info(String.format("Destroying %s", PersistenceService.class.getSimpleName()));
         logger.info("Persisting data before destroying");
-        this.persist(stateService.getCurrentPosts());
+        if (EventListenerBean.isInit()) {
+            this.persist(stateService.getCurrentPosts());
+        }
     }
 
-    public void persist(Map<String, Post> currentPosts) {
+    private void persist(Map<String, Post> currentPosts) {
 
-        try (PrintWriter out = new PrintWriter(VripperApplication.dataPath, "UTF-8")) {
+        try (PrintWriter out = new PrintWriter(dataPath, StandardCharsets.UTF_8)) {
             out.print(om.writeValueAsString(currentPosts));
         } catch (IOException e) {
             logger.error("Failed to persist app state", e);
@@ -95,6 +117,14 @@ public class PersistenceService {
             return om.readValue(content, om.getTypeFactory().constructMapType(HashMap.class, String.class, Post.class));
         } catch (IOException e) {
             logger.error("Failed to read app state", e);
+            long timestamp = new Date().getTime();
+            logger.warn(String.format("trying to rename old data file from %s to %s", dataPath, dataPath + "." + timestamp + ".old"));
+            try {
+                Files.move(new File(dataPath).toPath(), new File(dataPath + "." + timestamp + ".old").toPath());
+            } catch (IOException ex) {
+                logger.error(String.format("Failed to rename %s to %s", dataPath, dataPath + ".old"));
+                SpringContext.close();
+            }
         }
 
         return new HashMap<>();
@@ -102,12 +132,12 @@ public class PersistenceService {
 
     public void restore() {
 
-        String jsonContent;
+        String jsonContent = null;
         try {
-            jsonContent = Files.readAllLines(Paths.get(VripperApplication.dataPath), StandardCharsets.UTF_8).stream().collect(Collectors.joining());
+            jsonContent = String.join("", Files.readAllLines(Paths.get(dataPath), StandardCharsets.UTF_8));
         } catch (Exception e) {
-            logger.warn("data file not found, previous state cannot be restored", e);
-            return;
+            logger.error("data file cannot be read, previous state cannot be restored", e);
+            SpringContext.close();
         }
 
         Map<String, Post> read = read(jsonContent);
@@ -115,7 +145,7 @@ public class PersistenceService {
         stateService.getCurrentPosts().clear();
         stateService.getCurrentPosts().putAll(read);
         stateService.getCurrentPosts().values().forEach(p -> {
-            if(Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL).contains(p.getStatus())) {
+            if (Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL, Post.Status.PENDING).contains(p.getStatus())) {
                 p.setStatus(Post.Status.STOPPED);
             }
         });
@@ -126,8 +156,6 @@ public class PersistenceService {
             stateService.getCurrentImages().put(e.getUrl(), e);
         });
 
-        read.values().forEach(e -> {
-            e.setAppStateService(stateService);
-        });
+        read.values().forEach(e -> e.setAppStateService(stateService));
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -1,51 +1,27 @@
 package tn.mnlr.vripper.services;
 
-import io.reactivex.processors.PublishProcessor;
-import lombok.Getter;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
-import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.q.DownloadQ;
 
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-import java.io.BufferedInputStream;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
+import javax.annotation.PostConstruct;
 import java.util.List;
-import java.util.Optional;
 
 @Service
 public class PostParser {
 
     private static final Logger logger = LoggerFactory.getLogger(PostParser.class);
 
-    private static final String VR_API = "https://vipergirls.to/vr.php";
+    static final String VR_API = "https://vipergirls.to/vr.php";
 
     @Autowired
     private ConnectionManager cm;
 
-    SAXParserFactory factory = SAXParserFactory.newInstance();
-
-    @Autowired
-    private List<Host> supportedHosts;
-
     @Autowired
     private AppStateService appStateService;
 
@@ -61,20 +37,31 @@ public class PostParser {
     @Autowired
     private VipergirlsAuthService vipergirlsAuthService;
 
-    public PostParser() throws ParserConfigurationException, SAXException {
+    @Autowired
+    private List<Host> supportedHosts;
+
+    @PostConstruct
+    private void init() {
+
     }
 
-    public void addPost(String postId, String threadId) throws PostParseException {
+    public synchronized void addPost(String postId, String threadId) throws PostParseException {
 
-        VRPostParser vrPostParser = new VRPostParser(threadId, postId);
+        if (appStateService.getCurrentPosts().containsKey(postId)) {
+            logger.warn(String.format("skipping %s, already loaded", postId));
+            return;
+        }
+
+        VRPostParser vrPostParser = new VRPostParser(threadId, postId, cm, vipergirlsAuthService, supportedHosts);
         Post post = vrPostParser.parse();
 
         post.setAppStateService(appStateService);
         post.getImages().forEach(e -> e.setAppStateService(appStateService));
         authService.leaveThanks(post.getUrl(), post.getPostId());
         if (appSettingsService.isAutoStart()) {
-            logger.info("Auto start downloads option is enabled");
-            logger.info(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
+            logger.debug("Auto start downloads option is enabled");
+            logger.debug(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
+            post.setStatus(Post.Status.PENDING);
             try {
                 downloadQ.enqueue(post);
             } catch (InterruptedException e) {
@@ -82,297 +69,14 @@ public class PostParser {
                 Thread.currentThread().interrupt();
                 return;
             }
-            logger.info(String.format("Done enqueuing jobs for %s", post.getUrl()));
+            logger.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
         } else {
-            logger.info("Auto start downloads option is disabled");
+            post.setStatus(Post.Status.STOPPED);
+            logger.debug("Auto start downloads option is disabled");
         }
     }
 
-    @Getter
-    public static abstract class VRPostState {
-        private final String threadId;
-
-        protected VRPostState(String threadId) {
-            this.threadId = threadId;
-        }
-    }
-
-    @Getter
-    public static class VRPostParse extends VRPostState {
-
-        private final String type = "postParse";
-        private String postId;
-        private int number;
-        private String title;
-        private int imageCount;
-        private String url;
-        private List<String> previews;
-
-
-        public VRPostParse(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews) {
-            super(threadId);
-            this.postId = postId;
-            this.number = number;
-            this.title = title;
-            this.imageCount = imageCount;
-            this.previews = previews;
-            this.url = url;
-        }
-    }
-
-    @Getter
-    public static class VRThreadParseState extends VRPostState {
-
-        private final String type = "threadParseState";
-        private final String state;
-
-        public VRThreadParseState(String threadId, String state) {
-            super(threadId);
-            this.state = state;
-        }
-    }
-
-    public class VRThreadParser {
-
-        @Getter
-        private final PublishProcessor<VRPostState> postPublishProcessor = PublishProcessor.create();
-
-        private String threadId;
-
-        public VRThreadParser(String threadId) {
-            this.threadId = threadId;
-        }
-
-        public Void parse() throws PostParseException {
-
-            logger.info(String.format("Parsing thread %s", threadId));
-            HttpGet httpGet;
-            try {
-                URIBuilder uriBuilder = new URIBuilder(VR_API);
-                uriBuilder.setParameter("t", threadId);
-                httpGet = cm.buildHttpGet(uriBuilder.build());
-                if (vipergirlsAuthService.getCookies() != null) {
-                    httpGet.addHeader("Cookie", vipergirlsAuthService.getCookies());
-                }
-            } catch (URISyntaxException e) {
-                throw new PostParseException(e);
-            }
-            VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
-            HttpClient connection = cm.getClient().build();
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
-
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
-            }
-            return null;
-        }
-    }
-
-    private class VRThreadHandler extends DefaultHandler {
-
-        private final String threadId;
-        private final PublishProcessor<VRPostState> vrPostPublishProcessor;
-        private String threadTitle;
-        private String postId;
-        private String postTitle;
-        private int imageCount;
-        private int postCounter;
-        private int previewCounter = 0;
-        private List<String> previews = new ArrayList<>();
-
-        public VRThreadHandler(String threadId, PublishProcessor<VRPostState> vrPostPublishProcessor) {
-            this.vrPostPublishProcessor = vrPostPublishProcessor;
-            this.threadId = threadId;
-        }
-
-        @Override
-        public void startDocument() {
-            vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "START"));
-        }
-
-        @Override
-        public void startElement(String uri, String localName, String qName, Attributes attributes) {
-
-            switch (qName.toLowerCase()) {
-                case "thread":
-                    threadTitle = attributes.getValue("title").trim();
-                    break;
-                case "post":
-                    imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
-                    postId = attributes.getValue("id").trim();
-                    postCounter = Integer.parseInt(attributes.getValue("number").trim());
-                    postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
-                    break;
-                case "image":
-                    if (previewCounter++ < 4) {
-                        String thumbUrl = Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).orElse(null);
-                        if (thumbUrl != null) {
-                            previews.add(thumbUrl);
-                        }
-                    }
-                    break;
-            }
-        }
-
-        @Override
-        public void endElement(String uri, String localName, String qName) {
-            switch (qName.toLowerCase()) {
-                case "post":
-                    if (imageCount != 0) {
-                        vrPostPublishProcessor.onNext(new VRPostParse(
-                                threadId,
-                                postId,
-                                postCounter,
-                                postTitle,
-                                imageCount,
-                                String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
-                                previews
-                        ));
-                    }
-                    previewCounter = 0;
-                    previews = new ArrayList<>();
-                    break;
-            }
-        }
-
-        @Override
-        public void endDocument() {
-            vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "END"));
-            vrPostPublishProcessor.onComplete();
-        }
-    }
-
-    public class VRPostParser {
-
-        private String threadId;
-        private String postId;
-
-        public VRPostParser(String threadId, String postId) {
-            this.threadId = threadId;
-            this.postId = postId;
-        }
-
-        public Post parse() throws PostParseException {
-
-            Post post;
-            logger.info(String.format("Parsing post %s", postId));
-            HttpGet httpGet;
-            try {
-                URIBuilder uriBuilder = new URIBuilder(VR_API);
-                uriBuilder.setParameter("p", postId);
-                httpGet = cm.buildHttpGet(uriBuilder.build());
-                if (vipergirlsAuthService.getCookies() != null) {
-                    httpGet.addHeader("Cookie", vipergirlsAuthService.getCookies());
-                }
-            } catch (URISyntaxException e) {
-                throw new PostParseException(e);
-            }
-            VRPostHandler handler = new VRPostHandler(threadId, postId);
-            HttpClient connection = cm.getClient().build();
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
-
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                post = handler.getParsedPost();
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
-            }
-            return post;
-        }
-    }
-
-    private class VRPostHandler extends DefaultHandler {
-
-        private final String threadId;
-        private final String postId;
-        private String threadTitle;
-
-        private int previewCounter = 0;
-
-        private String postTitle;
-        private int imageCount;
-
-        private List<String> previews = new ArrayList<>();
-        private List<Image> images = new ArrayList<>();
-        @Getter
-        private Post parsedPost;
-
-        public VRPostHandler(String threadId, String postId) {
-            this.threadId = threadId;
-            this.postId = postId;
-        }
-
-        @Override
-        public void startDocument() {
-        }
-
-        @Override
-        public void startElement(String uri, String localName, String qName, Attributes attributes) {
-
-            switch (qName.toLowerCase()) {
-                case "thread":
-                    threadTitle = attributes.getValue("title").trim();
-                    break;
-                case "post":
-                    imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
-                    postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
-                    break;
-                case "image":
-                    if (previewCounter++ < 4) {
-                        String thumbUrl = Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).orElse(null);
-                        if (thumbUrl != null) {
-                            previews.add(thumbUrl);
-                        }
-                    }
-
-                    String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
-                    if (mainUrl != null) {
-                        Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
-                        if (foundHost != null) {
-                            logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
-                            images.add(new Image(mainUrl, postId, postTitle, foundHost, imageCount));
-                        } else {
-                            logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
-                        }
-                    }
-                    break;
-            }
-        }
-
-        @Override
-        public void endElement(String uri, String localName, String qName) {
-            switch (qName.toLowerCase()) {
-                case "post":
-                    if (imageCount != 0) {
-                        HashMap<String, Object> metadata = new HashMap<>();
-                        metadata.put("PREVIEWS", previews);
-                        parsedPost = new Post(
-                                postTitle,
-                                String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
-                                images,
-                                metadata,
-                                postId,
-                                threadId
-                        );
-                    }
-                    previewCounter = 0;
-                    previews = new ArrayList<>();
-                    images = new ArrayList<>();
-                    break;
-            }
-        }
+    public VRThreadParser createVRThreadParser(String threadId) {
+        return new VRThreadParser(threadId, cm, vipergirlsAuthService, supportedHosts);
     }
 }
-

vripper-server/src/main/java/tn/mnlr/vripper/services/QueuedVGLink.java

@@ -0,0 +1,38 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.util.Objects;
+
+@Getter
+@ToString
+public class QueuedVGLink {
+
+    private final String type = "grabQueue";
+    private final String link;
+    private final String threadId;
+    private final String postId;
+    @Setter
+    boolean removed;
+
+    public QueuedVGLink(String link, String threadId, String postId) {
+        this.link = link;
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        QueuedVGLink that = (QueuedVGLink) o;
+        return Objects.equals(link, that.link);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(link);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/ThumbnailGenerator.java

@@ -0,0 +1,161 @@
+package tn.mnlr.vripper.services;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.imgscalr.Scalr;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.util.FileSystemUtils;
+
+import javax.annotation.PostConstruct;
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Service
+public class ThumbnailGenerator {
+
+    @Getter
+    private LoadingCache<CacheKey, byte[]> thumbnails;
+
+    @Value("${base.dir}")
+    private String baseDir;
+
+    @Autowired
+    private PathService pathService;
+
+    @Getter
+    private File cacheFolder;
+    CacheLoader<CacheKey, byte[]> loader = new CacheLoader<>() {
+        @Override
+        public byte[] load(CacheKey key) throws Exception {
+            File destinationFolder = pathService.getDownloadDestinationFolder(key.getPostId());
+            if (!destinationFolder.exists() || !destinationFolder.isDirectory()) {
+                return null;
+            }
+            return Files.readAllBytes(generateThumbnail(new File(destinationFolder, key.getImgName()), key.getPostId()).toPath());
+        }
+    };
+
+    @PostConstruct
+    private void init() throws Exception {
+        cacheFolder = new File(baseDir + File.separator + ".vripper" + File.separator + "cache");
+        cacheFolder.mkdirs();
+        if (!cacheFolder.exists()) {
+            throw new Exception(String.format("%s could not be created", cacheFolder.toString()));
+        }
+        thumbnails = CacheBuilder.newBuilder()
+                .maximumSize(20000)
+                .build(loader);
+    }
+
+    public void clearCache() {
+        thumbnails.invalidateAll();
+        for (File file : Optional.ofNullable(cacheFolder.listFiles()).orElse(new File[]{})) {
+            FileSystemUtils.deleteRecursively(file);
+        }
+    }
+
+    public long cacheSize() {
+
+        final AtomicLong size = new AtomicLong(0);
+
+        try {
+            Files.walkFileTree(cacheFolder.toPath(), new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+
+                    size.addAndGet(attrs.size());
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+
+                    System.out.println("skipped: " + file + " (" + exc + ")");
+                    // Skip folders that can't be traversed
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+
+                    if (exc != null)
+                        System.out.println("had trouble traversing: " + dir + " (" + exc + ")");
+                    else
+                        size.addAndGet(dir.toFile().length());
+                    // Ignore errors traversing a folder
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new AssertionError("walkFileTree will not throw IOException if the FileVisitor does not");
+        }
+
+        return size.get();
+    }
+
+    private File generateThumbnail(File inputFile, String postId) throws Exception {
+        if (!inputFile.exists()) {
+            throw new Exception(String.format("Input file %s does not exist", inputFile.toString()));
+        }
+        File postsCacheFolder = new File(cacheFolder, postId);
+        postsCacheFolder.mkdirs();
+        if (!postsCacheFolder.exists()) {
+            throw new Exception(String.format("%s could not be created", postsCacheFolder.toString()));
+        }
+
+        File thumbFile = new File(postsCacheFolder, inputFile.getName());
+        if (thumbFile.exists()) {
+            return thumbFile;
+        }
+
+        BufferedImage image = ImageIO.read(inputFile);
+        BufferedImage resize = Scalr.resize(image, 350, 350);
+        ImageIO.write(resize, "jpg", thumbFile);
+        image.flush();
+        resize.flush();
+        return thumbFile;
+    }
+
+    @Getter
+    @Setter
+    @NoArgsConstructor
+    public static class CacheKey {
+        private String postId;
+        private String imgName;
+
+        public CacheKey(String postId, String imgName) {
+            this.postId = postId;
+            this.imgName = imgName;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            CacheKey cacheKey = (CacheKey) o;
+            return Objects.equals(postId, cacheKey.postId) &&
+                    Objects.equals(imgName, cacheKey.imgName);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(postId, imgName);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VGHandler.java

@@ -0,0 +1,83 @@
+package tn.mnlr.vripper.services;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.VripperApplication;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+@Service
+public class VGHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(VGHandler.class);
+
+    @Autowired
+    private AppStateService appStateService;
+
+    @Autowired
+    private PostParser postParser;
+
+    @Getter
+    private LoadingCache<String, List<VRPostState>> cache;
+
+    @PostConstruct
+    private void init() {
+
+        CacheLoader<String, List<VRPostState>> loader = new CacheLoader<>() {
+            @Override
+            public List<VRPostState> load(String threadId) throws Exception {
+                VRThreadParser vrThreadParser = postParser.createVRThreadParser(threadId);
+                List<VRPostState> posts = new ArrayList<>();
+                vrThreadParser.parse();
+                vrThreadParser.getPostPublishProcessor()
+                        .onBackpressureBuffer()
+                        .blockingSubscribe(posts::add);
+                return posts;
+            }
+        };
+
+        cache = CacheBuilder.newBuilder()
+                .expireAfterWrite(30, TimeUnit.MINUTES)
+                .build(loader);
+    }
+
+    public void handle(List<QueuedVGLink> queuedVGLinks) throws Exception {
+        for (QueuedVGLink queuedVGLink : queuedVGLinks) {
+            if (queuedVGLink.getPostId() != null) {
+                postParser.addPost(queuedVGLink.getPostId(), queuedVGLink.getThreadId());
+            } else {
+                Callable<Void> cl = () -> {
+                    List<VRPostState> vrPostStates = cache.get(queuedVGLink.getThreadId());
+                    logger.debug(String.format("%d found for %s", vrPostStates.size(), queuedVGLink.getLink()));
+                    if (vrPostStates.size() == 1) {
+                        postParser.addPost(vrPostStates.get(0).getPostId(), vrPostStates.get(0).getThreadId());
+                        logger.debug(String.format("threadId %s, postId %s is added automatically for download", queuedVGLink.getThreadId(), queuedVGLink.getPostId()));
+                    } else {
+                        appStateService.getGrabQueue().put(queuedVGLink.getLink(), queuedVGLink);
+                        appStateService.getLiveGrabQueue().onNext(queuedVGLink);
+                    }
+                    return null;
+                };
+                VripperApplication.commonExecutor.submit(cl);
+            }
+        }
+    }
+
+    public void remove(String url) {
+        Optional.ofNullable(appStateService.getGrabQueue().remove(url)).ifPresent(e -> {
+            e.setRemoved(true);
+            appStateService.getLiveGrabQueue().onNext(e);
+        });
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostParser.java

@@ -0,0 +1,190 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.exception.DownloadException;
+import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.host.Host;
+
+import javax.xml.parsers.SAXParserFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static tn.mnlr.vripper.services.PostParser.VR_API;
+
+class VRPostParser {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRPostParser.class);
+
+    private static SAXParserFactory factory = SAXParserFactory.newInstance();
+
+    private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+            .handleIf(e -> e instanceof IOException)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+            .abortOn(InterruptedException.class)
+            .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+
+    private final String threadId;
+    private final String postId;
+    private final ConnectionManager cm;
+    private final VipergirlsAuthService vipergirlsAuthService;
+    private final List<Host> supportedHosts;
+
+    VRPostParser(String threadId, String postId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> supportedHosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.cm = cm;
+        this.vipergirlsAuthService = vipergirlsAuthService;
+        this.supportedHosts = supportedHosts;
+    }
+
+    public Post parse() throws PostParseException {
+
+        logger.debug(String.format("Parsing post %s", postId));
+        HttpGet httpGet;
+        try {
+            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            uriBuilder.setParameter("p", postId);
+            httpGet = cm.buildHttpGet(uriBuilder.build());
+        } catch (URISyntaxException e) {
+            throw new PostParseException(e);
+        }
+        VRPostHandler handler = new VRPostHandler(threadId, postId, supportedHosts);
+        AtomicReference<Throwable> thr = new AtomicReference<>();
+        logger.debug(String.format("Requesting %s", httpGet));
+        Post post = Failsafe.with(retryPolicy)
+                .onFailure(e -> thr.set(e.getFailure()))
+                .get(() -> {
+                    HttpClient connection = cm.getClient().build();
+                    try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
+                        if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                            throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                        }
+
+                        try {
+                            factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                            return handler.getParsedPost();
+                        } catch (Exception e) {
+                            throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
+                        } finally {
+                            EntityUtils.consumeQuietly(response.getEntity());
+                        }
+                    }
+                });
+        if (thr.get() != null || post == null) {
+            logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
+            throw new PostParseException(thr.get());
+        }
+        return post;
+    }
+}
+
+class VRPostHandler extends DefaultHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRPostHandler.class);
+
+    private final List<Host> supportedHosts;
+    private final String threadId;
+    private final String postId;
+    private List<Image> images = new ArrayList<>();
+    private List<String> previews = new ArrayList<>();
+    private String threadTitle;
+    private String postTitle;
+    private String forum;
+    private int previewCounter = 0;
+    private int index = 0;
+    private int imageCount;
+
+    @Getter
+    private Post parsedPost;
+
+    VRPostHandler(String threadId, String postId, List<Host> supportedHosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.supportedHosts = supportedHosts;
+    }
+
+    @Override
+    public void startDocument() {
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) {
+
+        switch (qName.toLowerCase()) {
+            case "forum":
+                forum = attributes.getValue("title").trim();
+                break;
+            case "thread":
+                threadTitle = attributes.getValue("title").trim();
+                break;
+            case "post":
+                imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
+                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                break;
+            case "image":
+                index++;
+                if (previewCounter++ < 4) {
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
+                }
+
+                String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
+                if (mainUrl != null) {
+                    Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
+                    if (foundHost != null) {
+                        logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
+                        images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
+                    } else {
+                        logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
+                    }
+                }
+                break;
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName) {
+        if ("post".equals(qName.toLowerCase())) {
+            if (imageCount != 0) {
+                HashMap<String, Object> metadata = new HashMap<>();
+                metadata.put("PREVIEWS", previews);
+                parsedPost = new Post(
+                        postTitle,
+                        String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
+                        images,
+                        metadata,
+                        postId,
+                        threadId,
+                        threadTitle,
+                        forum
+                );
+            }
+            index = 0;
+            previewCounter = 0;
+            previews = new ArrayList<>();
+            images = new ArrayList<>();
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostState.java

@@ -0,0 +1,29 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+
+import java.util.List;
+
+@Getter
+public class VRPostState {
+    private final String type = "postParse";
+    private final String threadId;
+    private String postId;
+    private int number;
+    private String title;
+    private int imageCount;
+    private String url;
+    private List<String> previews;
+    private String hosts;
+
+    VRPostState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.number = number;
+        this.title = title;
+        this.imageCount = imageCount;
+        this.previews = previews;
+        this.url = url;
+        this.hosts = hosts;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRThreadParser.java

@@ -0,0 +1,176 @@
+package tn.mnlr.vripper.services;
+
+import io.reactivex.processors.ReplayProcessor;
+import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import tn.mnlr.vripper.exception.DownloadException;
+import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.host.Host;
+
+import javax.xml.parsers.SAXParserFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static tn.mnlr.vripper.services.PostParser.VR_API;
+
+public class VRThreadParser {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRThreadParser.class);
+
+    private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+            .handleIf(e -> e instanceof IOException)
+            .withDelay(1, 3, ChronoUnit.SECONDS)
+            .withMaxRetries(2)
+            .withMaxDuration(Duration.of(10, ChronoUnit.SECONDS))
+            .abortOn(InterruptedException.class)
+            .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+    private static SAXParserFactory factory = SAXParserFactory.newInstance();
+    @Getter
+    private final ReplayProcessor<VRPostState> postPublishProcessor = ReplayProcessor.create();
+    private final String threadId;
+    private final ConnectionManager cm;
+    private final VipergirlsAuthService vipergirlsAuthService;
+    private final List<Host> hosts;
+
+
+    VRThreadParser(String threadId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> hosts) {
+        this.threadId = threadId;
+        this.cm = cm;
+        this.vipergirlsAuthService = vipergirlsAuthService;
+        this.hosts = hosts;
+    }
+
+    public Void parse() throws PostParseException {
+
+        logger.debug(String.format("Parsing thread %s", threadId));
+        HttpGet httpGet;
+        try {
+            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            uriBuilder.setParameter("t", threadId);
+            httpGet = cm.buildHttpGet(uriBuilder.build());
+        } catch (URISyntaxException e) {
+            throw new PostParseException(e);
+        }
+
+        VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor, hosts);
+        AtomicReference<Throwable> thr = new AtomicReference<>();
+        logger.debug(String.format("Requesting %s", httpGet));
+        Failsafe.with(retryPolicy)
+                .onFailure(e -> thr.set(e.getFailure()))
+                .get(() -> {
+                    HttpClient connection = cm.getClient().build();
+                    try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
+                        if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                            throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                        }
+
+                        try {
+                            factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                        } catch (Exception e) {
+                            throw new PostParseException(String.format("Failed to parse thread %s", threadId), e);
+                        } finally {
+                            EntityUtils.consumeQuietly(response.getEntity());
+                        }
+                    }
+                    return null;
+                });
+        if (thr.get() != null) {
+            logger.error(String.format("parsing failed for thread %s", threadId), thr.get());
+            throw new PostParseException(thr.get());
+        }
+        return null;
+    }
+}
+
+class VRThreadHandler extends DefaultHandler {
+
+    private final String threadId;
+    private final ReplayProcessor<VRPostState> vrPostPublishProcessor;
+    private final List<Host> supportedHosts;
+    private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
+    private List<String> previews = new ArrayList<>();
+    private String threadTitle;
+    private String postId;
+    private String postTitle;
+    private int imageCount;
+    private int postCounter;
+    private int previewCounter = 0;
+
+    VRThreadHandler(String threadId, ReplayProcessor<VRPostState> vrPostPublishProcessor, List<Host> supportedHosts) {
+        this.vrPostPublishProcessor = vrPostPublishProcessor;
+        this.threadId = threadId;
+        this.supportedHosts = supportedHosts;
+    }
+
+    @Override
+    public void startDocument() {
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) {
+
+        switch (qName.toLowerCase()) {
+            case "thread":
+                threadTitle = attributes.getValue("title").trim();
+                break;
+            case "post":
+                imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
+                postId = attributes.getValue("id").trim();
+                postCounter = Integer.parseInt(attributes.getValue("number").trim());
+                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                break;
+            case "image":
+                Optional.ofNullable(attributes.getValue("main_url"))
+                        .map(String::trim)
+                        .flatMap(mainUrl -> supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
+                        .ifPresent(host -> Optional.ofNullable(hostMap.get(host)).ifPresentOrElse(AtomicInteger::incrementAndGet, () -> hostMap.put(host, new AtomicInteger(0))));
+                if (previewCounter++ < 4) {
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
+                }
+                break;
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName) {
+        if ("post".equals(qName.toLowerCase())) {
+            if (imageCount != 0) {
+                vrPostPublishProcessor.onNext(new VRPostState(
+                        threadId,
+                        postId,
+                        postCounter,
+                        postTitle,
+                        imageCount,
+                        String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
+                        previews,
+                        hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
+                ));
+            }
+            previewCounter = 0;
+            previews = new ArrayList<>();
+            hostMap.clear();
+        }
+    }
+
+    @Override
+    public void endDocument() {
+        vrPostPublishProcessor.onComplete();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java

@@ -7,6 +7,9 @@ import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
@@ -18,6 +21,7 @@ import org.w3c.dom.Document;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.VripperException;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -42,7 +46,7 @@ public class VipergirlsAuthService {
     private AppSettingsService appSettingsService;
 
     @Getter
-    private String cookies;
+    private HttpClientContext context = HttpClientContext.create();
 
     private boolean authenticated = false;
 
@@ -52,6 +56,16 @@ public class VipergirlsAuthService {
     @Getter
     private PublishProcessor<String> loggedInUser = PublishProcessor.create();
 
+    @PostConstruct
+    private void init() {
+        context.setCookieStore(new BasicCookieStore());
+        try {
+            authenticate();
+        } catch (VripperException e) {
+            logger.error("Cannot authenticate user with ViperGirls", e);
+        }
+    }
+
     @PreDestroy
     private void destroy() throws InterruptedException {
         logger.info("Shutting down VipergirlsAuthService");
@@ -65,8 +79,8 @@ public class VipergirlsAuthService {
         authenticated = false;
 
         if(!appSettingsService.isVLogin()) {
-            logger.warn("Authentication option is disabled");
-            cookies = null;
+            logger.debug("Authentication option is disabled");
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             return;
@@ -77,7 +91,7 @@ public class VipergirlsAuthService {
 
         if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
             logger.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
-            cookies = null;
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             return;
@@ -97,7 +111,7 @@ public class VipergirlsAuthService {
         try {
             postAuth.setEntity(new UrlEncodedFormEntity(params));
         } catch (Exception e) {
-            cookies = null;
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             throw new VripperException(e);
@@ -108,28 +122,19 @@ public class VipergirlsAuthService {
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(postAuth)) {
+        try (CloseableHttpResponse response = client.execute(postAuth, context)) {
 
             if (response.getStatusLine().getStatusCode() / 100 != 2) {
                 throw new VripperException(String.format("Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
             }
-            StringBuilder sb = new StringBuilder();
-            Arrays.asList(response.getHeaders("set-cookie"))
-                    .stream()
-                    .map(e -> e.getValue())
-                    .map(e -> e.split(";")[0])
-                    .filter(e -> e.startsWith("vg_lastactivity") || e.startsWith("vg_userid") || e.startsWith("vg_password"))
-                    .forEach(e -> sb.append(e).append(";"));
-            String cookies = sb.toString();
             String responseBody = EntityUtils.toString(response.getEntity());
             logger.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
             EntityUtils.consumeQuietly(response.getEntity());
-            if(!cookies.contains("vg_userid")) {
+            if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) {
                 throw new VripperException("Failed to authenticate user with ViperRipper");
             }
-            this.cookies = cookies;
         } catch (Exception e) {
-            cookies = null;
+            context.getCookieStore().clear();
             loggedUser = "";
             loggedInUser.onNext(loggedUser);
             if (e instanceof VripperException) {
@@ -144,14 +149,14 @@ public class VipergirlsAuthService {
         loggedInUser.onNext(loggedUser);
     }
 
-    public void leaveThanks(String postUrl, String postId) {
+    void leaveThanks(String postUrl, String postId) {
         VripperApplication.commonExecutor.submit(() -> {
             if (!appSettingsService.isVLogin()) {
-                logger.warn("Authentication with ViperGirls option is disabled");
+                logger.debug("Authentication with ViperGirls option is disabled");
                 return;
             }
             if (!appSettingsService.isVThanks()) {
-                logger.warn("Leave thanks option is disabled");
+                logger.debug("Leave thanks option is disabled");
                 return;
             }
             if (!authenticated) {
@@ -168,16 +173,15 @@ public class VipergirlsAuthService {
 
     private String getSecurityToken(String url) throws VripperException {
 
-        String securityToken = "";
+        String securityToken;
 
         HttpGet httpGet = cm.buildHttpGet(url);
         httpGet.addHeader("Referer", "https://vipergirls.to/");
         httpGet.addHeader("Host", "vipergirls.to");
-        httpGet.addHeader("Cookie", cookies);
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(httpGet)) {
+        try (CloseableHttpResponse response = client.execute(httpGet, context)) {
 
             String postPage = EntityUtils.toString(response.getEntity());
             Document document = htmlProcessorService.clean(postPage);
@@ -189,7 +193,7 @@ public class VipergirlsAuthService {
                     .getTextContent()
                     .trim();
 
-            securityToken = Arrays.asList(thanksUrl.split("&amp;")).stream()
+            securityToken = Arrays.stream(thanksUrl.split("&amp;"))
                     .filter(v -> v.startsWith("securitytoken"))
                     .findAny()
                     .orElse("")
@@ -218,11 +222,10 @@ public class VipergirlsAuthService {
 
         postThanks.addHeader("Referer", "https://vipergirls.to/");
         postThanks.addHeader("Host", "vipergirls.to");
-        postThanks.addHeader("Cookie", cookies);
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(postThanks)) {
+        try (CloseableHttpResponse response = client.execute(postThanks, context)) {
             EntityUtils.consumeQuietly(response.getEntity());
         } catch (Exception e) {
             throw new VripperException(e);

vripper-server/src/main/java/tn/mnlr/vripper/services/XpathService.java

@@ -2,7 +2,6 @@ package tn.mnlr.vripper.services;
 
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 import tn.mnlr.vripper.exception.XpathException;
 
 import javax.xml.xpath.XPath;
@@ -12,20 +11,8 @@ import javax.xml.xpath.XPathFactory;
 @Service
 public class XpathService {
 
-    private XpathService() {
-
-    }
-
     private XPath xpath = XPathFactory.newInstance().newXPath();
 
-    public NodeList getAsNodeList(Node source, String xpathExpression) throws XpathException {
-        try {
-            return (NodeList) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET);
-        } catch (Exception e) {
-            throw new XpathException(e);
-        }
-    }
-
     public Node getAsNode(Node source, String xpathExpression) throws XpathException {
         try {
             return (Node) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE);

vripper-server/src/main/java/tn/mnlr/vripper/web/WebSocketConfig.java

@@ -16,6 +16,6 @@ public class WebSocketConfig implements WebSocketConfigurer {
 
     @Override
     public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
-        registry.addHandler(handler, "/endpoint").setAllowedOrigins("*").withSockJS();
+        registry.addHandler(handler, "/endpoint").setAllowedOrigins("*").withSockJS().setClientLibraryUrl("../../assets/sockjs.min.js");
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/GalleryEndpoint.java

@@ -0,0 +1,202 @@
+package tn.mnlr.vripper.web.restendpoints;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.CacheControl;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import tn.mnlr.vripper.services.AppSettingsService;
+import tn.mnlr.vripper.services.PathService;
+import tn.mnlr.vripper.services.ThumbnailGenerator;
+
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+import javax.servlet.http.HttpServletResponse;
+import java.awt.*;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+@RestController
+@CrossOrigin(value = "*")
+public class GalleryEndpoint {
+
+    private static final Logger logger = LoggerFactory.getLogger(GalleryEndpoint.class);
+
+    @Autowired
+    private PathService pathService;
+
+    @Autowired
+    private ThumbnailGenerator thumbnailGenerator;
+
+    private static final String cacheControl = CacheControl.maxAge(365, TimeUnit.DAYS).getHeaderValue();
+    @Autowired
+    private AppSettingsService appSettingsService;
+
+    @ExceptionHandler(Exception.class)
+    public ResponseEntity handleException(Exception e) {
+        logger.error("Error when process request", e);
+        return ResponseEntity
+                .status(HttpStatus.INTERNAL_SERVER_ERROR)
+                .body(e.getMessage());
+    }
+
+    @GetMapping(value = "/image/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<byte[]> getImage(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        response.addHeader("Cache-Control", cacheControl);
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        return ResponseEntity.ok(Files.readAllBytes(Paths.get(destinationFolder.toPath().toString(), imgName)));
+    }
+
+    @GetMapping(value = "/image/thumb/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<byte[]> getImageThumb(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        response.addHeader("Cache-Control", cacheControl);
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        if (destinationFolder.exists() && destinationFolder.isDirectory()) {
+            return ResponseEntity.ok(thumbnailGenerator.getThumbnails().get(new ThumbnailGenerator.CacheKey(postId, imgName)));
+        }
+        return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
+    }
+
+    @GetMapping("/gallery/{postId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<List<GalleryImage>> getGallery(@PathVariable("postId") @NonNull String postId) {
+        if (!appSettingsService.isViewPhotos()) {
+            return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
+        }
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        if (destinationFolder.exists() && destinationFolder.isDirectory()) {
+            return ResponseEntity.ok(
+                    Arrays.stream(Objects.requireNonNull(destinationFolder.listFiles()))
+                            .filter(f -> !f.getName().endsWith("tmp"))
+                            .filter(f -> f.getName().toLowerCase().endsWith(".jpg") || f.getName().toLowerCase().endsWith(".jpeg"))
+                            .sorted(Comparator.comparing(File::getName))
+                            .map(GalleryImage::fromFile)
+                            .filter(Objects::nonNull)
+                            .collect(Collectors.toList())
+            );
+        }
+        return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
+    }
+
+    @GetMapping("/gallery/cache")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> getCacheSize() {
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    @GetMapping("/gallery/cache/clear")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<CacheSize> clearCache() {
+        thumbnailGenerator.clearCache();
+        return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
+    }
+
+    private String humanReadableByteCount(long bytes, boolean si) {
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class CacheSize {
+
+    private String size;
+
+    public CacheSize(String size) {
+        this.size = size;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class GalleryImage {
+
+    private static final CacheLoader<File, Dimension> loader = new CacheLoader<>() {
+
+        @Override
+        public Dimension load(File file) {
+            try (ImageInputStream in = ImageIO.createImageInputStream(file)) {
+                final Iterator<ImageReader> readers = ImageIO.getImageReaders(in);
+                if (readers.hasNext()) {
+                    ImageReader reader = readers.next();
+                    try {
+                        reader.setInput(in);
+                        return new Dimension(reader.getWidth(0), reader.getHeight(0));
+                    } finally {
+                        reader.dispose();
+                    }
+                } else {
+                    logger.error(String.format("No reader found for image %s", file.toString()));
+                    return null;
+                }
+            } catch (Exception e) {
+                logger.error(String.format("Failed to create image object for %s", file.toString()), e);
+                return null;
+            }
+        }
+    };
+
+    private static final LoadingCache<File, Dimension> cache = CacheBuilder.newBuilder()
+            .maximumSize(20000)
+            .build(loader);
+
+    private static final Logger logger = LoggerFactory.getLogger(GalleryImage.class);
+    private String title;
+    private String src;
+    private String msrc;
+    private double w;
+    private double h;
+
+    public GalleryImage(String title, String src, String msrc, double w, double h) {
+        this.title = title;
+        this.src = src;
+        this.msrc = msrc;
+        this.w = w;
+        this.h = h;
+    }
+
+    public static GalleryImage fromFile(File file) {
+
+        Dimension dimension = null;
+        try {
+            dimension = cache.get(file);
+        } catch (ExecutionException e) {
+            logger.error(String.format("Failed to get image dimensions for %s", file.toString()), e);
+        }
+        if (dimension == null) {
+            return null;
+        }
+        return new GalleryImage(file.getName(), file.getName(), file.getName(), dimension.getWidth(), dimension.getHeight());
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java

@@ -1,7 +1,6 @@
 package tn.mnlr.vripper.web.restendpoints;
 
-import lombok.Getter;
-import lombok.ToString;
+import lombok.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -10,16 +9,17 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.PostParseException;
-import tn.mnlr.vripper.q.DownloadQ;
-import tn.mnlr.vripper.services.AppSettingsService;
-import tn.mnlr.vripper.services.AppStateService;
-import tn.mnlr.vripper.services.PathService;
-import tn.mnlr.vripper.services.PostParser;
+import tn.mnlr.vripper.q.ExecutionService;
+import tn.mnlr.vripper.services.*;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 @RestController
 @CrossOrigin(value = "*")
@@ -27,8 +27,7 @@ public class PostRestEndpoint {
 
     private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
 
-
-    private static final Pattern VG_URL_PATTERN = Pattern.compile("https:\\/\\/vipergirls\\.to\\/threads\\/(\\d+)((.*p=)(\\d+))?");
+    private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?");
 
     @Autowired
     private AppStateService appStateService;
@@ -39,8 +38,11 @@ public class PostRestEndpoint {
     @Autowired
     private PathService pathService;
 
+    @Autowired
+    private VGHandler vgHandler;
+
     @ExceptionHandler(Exception.class)
-    public ResponseEntity handleException(Exception e) {
+    public ResponseEntity<String> handleException(Exception e) {
         logger.error("Error when process request", e);
         return ResponseEntity
                 .status(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -48,51 +50,62 @@ public class PostRestEndpoint {
     }
 
     @Autowired
-    private DownloadQ downloadQ;
+    private ExecutionService executionService;
+
 
     @Autowired
     private PostParser postParser;
 
     @PostMapping("/post")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception {
-        logger.info(String.format("Starting to process thread: %s", url.url));
-        if (url.url == null || url.url.isEmpty()) {
-            return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST);
-        } else if (!url.url.startsWith("https://vipergirls.to")) {
-            return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
+    public synchronized ResponseEntity processPost(@RequestBody ThreadUrl _url) throws Exception {
+        if (_url.getUrl() == null || _url.getUrl().isEmpty()) {
+            return new ResponseEntity("Failed to process empty request", HttpStatus.BAD_REQUEST);
         }
-
-        String threadId, postId;
-        try {
-            Matcher m = VG_URL_PATTERN.matcher(url.url);
-            if (m.find()) {
-                threadId = m.group(1);
-                postId = m.group(4);
-            } else {
-                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+        List<String> urls = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
+        ArrayList<QueuedVGLink> queuedVGLinks = new ArrayList<>();
+        for (String url : urls) {
+            logger.debug(String.format("Starting to process thread: %s", url));
+            if (!url.startsWith("https://vipergirls.to")) {
+                logger.error(String.format("Unsupported link %s", url));
+                continue;
             }
-        } catch (Exception e) {
-            throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+
+            String threadId, postId;
+            try {
+                Matcher m = VG_URL_PATTERN.matcher(url);
+                if (m.find()) {
+                    threadId = m.group(1);
+                    postId = m.group(4);
+                } else {
+                    throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+                }
+            } catch (Exception e) {
+                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+            }
+            queuedVGLinks.add(new QueuedVGLink(url, threadId, postId));
         }
-        return ResponseEntity.ok(new PairThreadIdPostId(threadId, postId));
+        vgHandler.handle(queuedVGLinks);
+        return ResponseEntity.ok().build();
     }
 
     @PostMapping("/post/restart")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost(@RequestBody PostId postId) throws Exception {
-        downloadQ.restart(postId.getPostId());
+    public synchronized void restartPost(@RequestBody @NonNull List<PostId> postIds) throws Exception {
+        for (PostId postId : postIds) {
+            executionService.restart(postId.getPostId());
+        }
     }
 
     @PostMapping("/post/add")
     @ResponseStatus(value = HttpStatus.OK)
-    public void addPost(@RequestBody List<PostToAdd> posts) {
+    public synchronized void addPost(@RequestBody List<PostToAdd> posts) {
         for (PostToAdd post : posts) {
             VripperApplication.commonExecutor.submit(() -> {
                 try {
-                    postParser.addPost(post.postId, post.threadId);
+                    postParser.addPost(post.getPostId(), post.getThreadId());
                 } catch (PostParseException e) {
-                    logger.error(String.format("Failed to add post %s", post.postId), e);
+                    logger.error(String.format("Failed to add post %s", post.getPostId()), e);
                 }
             });
         }
@@ -107,128 +120,166 @@ public class PostRestEndpoint {
 
     @PostMapping("/post/restart/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost() throws Exception {
-        downloadQ.restartAll();
+    public synchronized void restartPost() throws Exception {
+        executionService.restartAll();
     }
 
     @PostMapping("/post/stop")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stop(@RequestBody PostId postId) {
-        downloadQ.stop(postId.getPostId());
+    public synchronized void stop(@RequestBody @NonNull List<PostId> postIds) {
+        for (PostId postId : postIds) {
+            executionService.stop(postId.getPostId());
+        }
     }
 
     @PostMapping("/post/stop/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stopAll() {
-        downloadQ.stopAll();
+    public synchronized void stopAll() {
+        executionService.stopAll();
     }
 
     @PostMapping("/post/remove")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity remove(@RequestBody PostId postId) {
-        downloadQ.stop(postId.getPostId());
-        appStateService.remove(postId.getPostId());
-        return ResponseEntity.ok(new RemoveResult(postId.getPostId()));
+    public synchronized ResponseEntity<List<RemoveResult>> remove(@RequestBody @NonNull List<PostId> postIds) {
+        List<RemoveResult> result = new ArrayList<>();
+        for (PostId postId : postIds) {
+            executionService.stop(postId.getPostId());
+            appStateService.remove(postId.getPostId());
+            result.add(new RemoveResult(postId.getPostId()));
+        }
+        return ResponseEntity.ok(result);
     }
 
     @PostMapping("/post/clear/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity clearAll() {
+    public synchronized ResponseEntity<RemoveAllResult> clearAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.clearAll()));
     }
 
     @PostMapping("/post/remove/all")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity removeAll() {
+    public synchronized ResponseEntity<RemoveAllResult> removeAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.removeAll()));
     }
 
-    @Getter
-    @ToString
-    private static class ThreadUrl {
-        private String url;
+    @GetMapping("/grab/{threadId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public synchronized ResponseEntity<List<VRPostState>> grab(@PathVariable("threadId") @NonNull ThreadId threadId) throws Exception {
+        return ResponseEntity.ok(vgHandler.getCache().get(threadId.getThreadId()));
     }
 
-    @Getter
-    private static class PostId {
-        private String postId;
-    }
-
-    @Getter
-    private static class PairThreadIdPostId {
-        private String threadId;
-        private String postId;
-
-        public PairThreadIdPostId(String threadId, String postId) {
-            this.threadId = threadId;
-            this.postId = postId;
-        }
-    }
-
-    @Getter
-    private static class PostToAdd {
-        private String threadId;
-        private String postId;
-    }
-
-    @Getter
-    private static class ParseResult {
-
-        private List<PostResult> posts;
-        private int count;
-        private String threadId;
-
-        public ParseResult(List<PostResult> posts, int count, String threadId) {
-            this.posts = posts;
-            this.count = count;
-            this.threadId = threadId;
-        }
-
-        @Getter
-        public static class PostResult {
-
-            private String title;
-            private int counter;
-            private String url;
-            private String postId;
-            private List<String> previews;
-
-            public PostResult(String title, int counter, String url, String postId, List<String> previews) {
-                this.title = title;
-                this.counter = counter;
-                this.url = url;
-                this.postId = postId;
-                this.previews = previews;
-            }
-        }
-    }
-
-    @Getter
-    private static class RemoveAllResult {
-        private List<String> postIds;
-        private int removed;
-
-        RemoveAllResult(List<String> postIds) {
-            this.removed = postIds.size();
-            this.postIds = postIds;
-        }
-    }
-
-    @Getter
-    private static class RemoveResult {
-        private String postId;
-
-        RemoveResult(String postId) {
-            this.postId = postId;
-        }
-    }
-
-    @Getter
-    private static class DownloadPath {
-        private String path;
-
-        DownloadPath(String path) {
-            this.path = path;
-        }
+    @PostMapping("/grab/remove")
+    @ResponseStatus(value = HttpStatus.OK)
+    public synchronized void grabRemove(@RequestBody @NonNull ThreadUrl threadUrl) {
+        vgHandler.remove(threadUrl.getUrl());
     }
 }
+
+@Getter
+@Setter
+@NoArgsConstructor
+class ThreadUrl {
+    private String url;
+
+    public ThreadUrl(String url) {
+        this.url = url;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class ThreadId {
+    private String threadId;
+
+    public ThreadId(String threadId) {
+        this.threadId = threadId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PostId {
+    private String postId;
+
+    public PostId(String postId) {
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PostToAdd {
+    private String threadId;
+    private String postId;
+
+    public PostToAdd(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class RemoveAllResult {
+    private List<String> postIds;
+    private int removed;
+
+    RemoveAllResult(List<String> postIds) {
+        this.removed = postIds.size();
+        this.postIds = postIds;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class RemoveResult {
+    private String postId;
+
+    RemoveResult(String postId) {
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class DownloadPath {
+    private String path;
+
+    DownloadPath(String path) {
+        this.path = path;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+@ToString
+class PairThreadIdPostId {
+    private String threadId;
+    private String postId;
+
+    public PairThreadIdPostId(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        PairThreadIdPostId that = (PairThreadIdPostId) o;
+        return Objects.equals(threadId, that.threadId) &&
+                Objects.equals(postId, that.postId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(threadId, postId);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -31,37 +31,62 @@ public class SettingsRestEndpoint {
                 .body(e.getMessage());
     }
 
+    @PostMapping("/settings/theme")
+    @ResponseStatus(value = HttpStatus.OK)
+    public AppSettingsService.Theme postTheme(@RequestBody AppSettingsService.Theme theme) {
+        synchronized (this.settings) {
+            this.settings.setTheme(theme);
+            return settings.getTheme();
+        }
+    }
+
+    @GetMapping("/settings/theme")
+    @ResponseStatus(value = HttpStatus.OK)
+    public AppSettingsService.Theme getTheme() {
+        return settings.getTheme();
+    }
+
     @PostMapping("/settings")
     @ResponseStatus(value = HttpStatus.OK)
     public ResponseEntity postSettings(@RequestBody AppSettingsService.Settings settings) throws Exception {
 
-        try {
-            this.settings.check(settings);
-        } catch (ValidationException e) {
-            return new ResponseEntity(new Response(e.getMessage()), HttpStatus.BAD_REQUEST);
-        }
-        this.settings.setDownloadPath(settings.getDownloadPath());
-        this.settings.setMaxThreads(settings.getMaxThreads());
-        this.settings.setAutoStart(settings.isAutoStart());
-        this.settings.setVLogin(settings.isVLogin());
-
-        if(settings.isVLogin()) {
-            this.settings.setVUsername(settings.getVUsername());
-            if (!this.settings.getVPassword().equals(settings.getVPassword())) {
-                this.settings.setVPassword(settings.getVPassword());
+        synchronized (this.settings) {
+            try {
+                this.settings.check(settings);
+            } catch (ValidationException e) {
+                return ResponseEntity
+                        .status(HttpStatus.BAD_REQUEST)
+                        .body(e.getMessage());
             }
-            this.settings.setVThanks(settings.isVThanks());
-        } else {
-            this.settings.setVUsername("");
-            this.settings.setVPassword("");
-            this.settings.setVThanks(false);
+            this.settings.setDownloadPath(settings.getDownloadPath());
+            this.settings.setMaxThreads(settings.getMaxThreads());
+            this.settings.setMaxTotalThreads(settings.getMaxTotalThreads());
+            this.settings.setAutoStart(settings.isAutoStart());
+            this.settings.setVLogin(settings.isVLogin());
+
+            if (settings.isVLogin()) {
+                this.settings.setVUsername(settings.getVUsername());
+                if (!this.settings.getVPassword().equals(settings.getVPassword())) {
+                    this.settings.setVPassword(settings.getVPassword());
+                }
+                this.settings.setVThanks(settings.isVThanks());
+            } else {
+                this.settings.setVUsername("");
+                this.settings.setVPassword("");
+                this.settings.setVThanks(false);
+            }
+            this.settings.setDesktopClipboard(settings.isDesktopClipboard());
+            this.settings.setForceOrder(settings.isForceOrder());
+            this.settings.setSubLocation(settings.isSubLocation());
+            this.settings.setThreadSubLocation(settings.isThreadSubLocation());
+            this.settings.setClearCompleted(settings.isClearCompleted());
+            this.settings.setViewPhotos(settings.isViewPhotos());
+            this.settings.setNotificationEnabled(settings.isNotification());
+
+            this.settings.save();
+
+            vipergirlsAuthService.authenticate();
         }
-        this.settings.setDesktopClipboard(settings.isDesktopClipboard());
-        this.settings.setForceOrder(settings.isForceOrder());
-
-        this.settings.save();
-
-        vipergirlsAuthService.authenticate();
         return ResponseEntity.ok(getSettings());
     }
 
@@ -72,21 +97,27 @@ public class SettingsRestEndpoint {
         return new AppSettingsService.Settings(
                 settings.getDownloadPath(),
                 settings.getMaxThreads(),
+                settings.getMaxTotalThreads(),
                 settings.isAutoStart(),
                 settings.isVLogin(),
                 settings.getVUsername(),
                 settings.getVPassword(),
                 settings.isVThanks(),
                 settings.isDesktopClipboard(),
-                settings.isForceOrder()
+                settings.isForceOrder(),
+                settings.isSubLocation(),
+                settings.isThreadSubLocation(),
+                settings.isClearCompleted(),
+                settings.isViewPhotos(),
+                settings.isNotificationEnabled()
         );
     }
 
     @Getter
-    class Response {
+    static class Response {
         String message;
 
-        public Response(String message) {
+        Response(String message) {
             this.message = message;
         }
     }

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java

@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.schedulers.Schedulers;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,7 +14,6 @@ import org.springframework.web.socket.CloseStatus;
 import org.springframework.web.socket.TextMessage;
 import org.springframework.web.socket.WebSocketSession;
 import org.springframework.web.socket.handler.TextWebSocketHandler;
-import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.ui.ImageUIMixin;
@@ -20,7 +21,7 @@ import tn.mnlr.vripper.entities.mixin.ui.PostUIMixin;
 import tn.mnlr.vripper.services.*;
 
 import java.io.IOException;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
@@ -54,10 +55,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private Map<String, Disposable> postsSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> postDetailsSubscriptions = new ConcurrentHashMap<>();
-    private Map<String, Disposable> vrPostParserSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> stateSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> downloadSpeedSubscriptions = new ConcurrentHashMap<>();
     private Map<String, Disposable> userSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> grabQueueSubscriptions = new ConcurrentHashMap<>();
 
     private Map<String, Future<Void>> threadParseRequests = new ConcurrentHashMap<>();
 
@@ -69,6 +70,9 @@ public class WebSocketHandler extends TextWebSocketHandler {
         WSMessage wsMessage = om.readValue(message.getPayload(), WSMessage.class);
         WSMessage.CMD cmd = WSMessage.CMD.valueOf(wsMessage.getCmd());
         switch (cmd) {
+            case GRAB_QUEUE_SUB:
+                subscribeForGrabQueue(session);
+                break;
             case GLOBAL_STATE_SUB:
                 subscribeForGlobalState(session);
                 break;
@@ -81,49 +85,45 @@ public class WebSocketHandler extends TextWebSocketHandler {
             case POSTS_SUB:
                 subscribeForPosts(session);
                 break;
-            case THREAD_PARSING_SUB:
-                subscribeForThreadParsing(session, wsMessage.getPayload());
-                break;
             case POST_DETAILS_SUB:
                 subscribeForPostDetails(session, wsMessage.getPayload());
                 break;
             case POST_DETAILS_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from post details", session.getId()));
-                Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-                break;
-            case THREAD_PARSING_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from thread parsing", session.getId()));
-                Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-                Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
+                logger.debug(String.format("Client %s unsubscribed from post details", session.getId()));
+                Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case POSTS_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from posts", session.getId()));
-                Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from posts", session.getId()));
+                Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case GLOBAL_STATE_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from global state", session.getId()));
-                Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from global state", session.getId()));
+                Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case SPEED_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from download speed info", session.getId()));
-                Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from download speed info", session.getId()));
+                Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
             case USER_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from user info", session.getId()));
-                Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from user info", session.getId()));
+                Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
+            case GRAB_QUEUE_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from grab queue", session.getId()));
+                Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
         }
     }
 
     private void subscribeForGlobalState(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for global state", session.getId()));
+        logger.debug(String.format("Client %s subscribed for global state", session.getId()));
         if (stateSubscriptions.containsKey(session.getId())) {
             stateSubscriptions.get(session.getId()).dispose();
         }
 
         try {
-            send(session, new TextMessage(om.writeValueAsString(Arrays.asList(globalStateService.getCurrentState()))));
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(globalStateService.getCurrentState()))));
         } catch (Exception e) {
             logger.error("Unexpected error occurred", e);
         }
@@ -143,13 +143,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private void subscribeForSpeed(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for download speed info", session.getId()));
+        logger.debug(String.format("Client %s subscribed for download speed info", session.getId()));
         if (downloadSpeedSubscriptions.containsKey(session.getId())) {
             downloadSpeedSubscriptions.get(session.getId()).dispose();
         }
 
         try {
-            send(session, new TextMessage(om.writeValueAsString(Arrays.asList(new DownloadSpeed(0)))));
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new DownloadSpeed(0)))));
         } catch (Exception e) {
             logger.error("Unexpected error occurred", e);
         }
@@ -170,13 +170,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private void subscribeForUser(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for user info", session.getId()));
+        logger.debug(String.format("Client %s subscribed for user info", session.getId()));
         if (userSubscriptions.containsKey(session.getId())) {
             userSubscriptions.get(session.getId()).dispose();
         }
 
         try {
-            send(session, new TextMessage(om.writeValueAsString(Arrays.asList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
         } catch (Exception e) {
             logger.error("Unexpected error occurred", e);
         }
@@ -185,7 +185,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 vipergirlsAuthService.getLoggedInUser()
                         .onBackpressureBuffer()
                         .observeOn(Schedulers.io())
-                        .map(e -> Arrays.asList(new LoggedUser(e)))
+                        .map(e -> Collections.singletonList(new LoggedUser(e)))
                         .map(om::writeValueAsString)
                         .map(TextMessage::new)
                         .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
@@ -194,7 +194,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private void subscribeForPosts(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for posts", session.getId()));
+        logger.debug(String.format("Client %s subscribed for posts", session.getId()));
         if (postsSubscriptions.containsKey(session.getId())) {
             postsSubscriptions.get(session.getId()).dispose();
         }
@@ -209,7 +209,6 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 appStateService.getLivePostsState()
                         .onBackpressureBuffer()
                         .observeOn(Schedulers.io())
-                        .filter(e -> !e.isRemoved())
                         .buffer(2000, TimeUnit.MILLISECONDS, 200)
                         .filter(e -> !e.isEmpty())
                         .map(e -> e.stream().distinct().collect(Collectors.toList()))
@@ -219,35 +218,35 @@ public class WebSocketHandler extends TextWebSocketHandler {
         );
     }
 
-    private void subscribeForThreadParsing(WebSocketSession session, String threadId) {
+    private void subscribeForGrabQueue(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for thread parsing with threadId = %s", session.getId(), threadId));
-        if (vrPostParserSubscriptions.containsKey(session.getId())) {
-            vrPostParserSubscriptions.get(session.getId()).dispose();
+        logger.debug(String.format("Client %s subscribed for grab queue", session.getId()));
+        if (grabQueueSubscriptions.containsKey(session.getId())) {
+            grabQueueSubscriptions.get(session.getId()).dispose();
         }
 
-        if (threadParseRequests.containsKey(session.getId())) {
-            threadParseRequests.get(session.getId()).cancel(true);
+        try {
+            send(session, new TextMessage(om.writeValueAsString(appStateService.getGrabQueue().values())));
+        } catch (Exception e) {
+            logger.error("Unexpected error occurred", e);
         }
 
-        PostParser.VRThreadParser vrThreadParser = postParser.new VRThreadParser(threadId);
-
-        vrPostParserSubscriptions.put(session.getId(), vrThreadParser.getPostPublishProcessor()
-                .onBackpressureBuffer()
-                .observeOn(Schedulers.io())
-                .buffer(200)
-                .filter(e -> !e.isEmpty())
-                .map(om::writeValueAsString)
-                .map(TextMessage::new)
-                .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        grabQueueSubscriptions.put(session.getId(),
+                appStateService.getLiveGrabQueue()
+                        .onBackpressureBuffer()
+                        .observeOn(Schedulers.io())
+                        .buffer(2000, TimeUnit.MILLISECONDS, 200)
+                        .filter(e -> !e.isEmpty())
+                        .map(e -> e.stream().distinct().collect(Collectors.toList()))
+                        .map(om::writeValueAsString)
+                        .map(TextMessage::new)
+                        .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
         );
-
-        threadParseRequests.put(session.getId(), VripperApplication.commonExecutor.submit(vrThreadParser::parse));
     }
 
     private void subscribeForPostDetails(WebSocketSession session, String postId) {
 
-        logger.info(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
+        logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
         if (postDetailsSubscriptions.containsKey(session.getId())) {
             postDetailsSubscriptions.get(session.getId()).dispose();
         }
@@ -283,22 +282,26 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     @Override
     public void afterConnectionEstablished(WebSocketSession session) {
-
+        logger.debug(String.format("Connection open for client id: %s", session.getId()));
     }
 
     @Override
     public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
 
-        Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+        Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
         Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
+
+        logger.debug(String.format("Connection closed for client id: %s", session.getId()));
     }
 
     @Getter
+    @Setter
+    @NoArgsConstructor
     private static class WSMessage {
 
         private String cmd;
@@ -313,10 +316,10 @@ public class WebSocketHandler extends TextWebSocketHandler {
             GLOBAL_STATE_UNSUB,
             SPEED_SUB,
             SPEED_UNSUB,
-            THREAD_PARSING_SUB,
-            THREAD_PARSING_UNSUB,
             USER_SUB,
-            USER_UNSUB
+            USER_UNSUB,
+            GRAB_QUEUE_SUB,
+            GRAB_QUEUE_UNSUB
         }
     }
 
@@ -326,7 +329,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
         private final String type = "user";
         private String user;
 
-        public LoggedUser(String user) {
+        LoggedUser(String user) {
             this.user = user;
         }
     }

vripper-server/src/main/resources/application-installer.properties

@@ -0,0 +1 @@
+base.dir=${user.home}

vripper-server/src/main/resources/application-portable.properties

@@ -0,0 +1 @@
+base.dir=${user.dir}

vripper-server/src/main/resources/application.properties

@@ -1,8 +1,10 @@
 logging.level.org.springframework.web=INFO
 logging.level.root=INFO
 logging.level.org.apache.http=INFO
-logging.file=${user.dir}/vripper.log
+logging.file=${base.dir}/.vripper/vripper.log
 server.port=${vripper.server.port:8080}
 management.endpoints.web.exposure.include=shutdown
 management.endpoint.shutdown.enabled=true
-endpoints.shutdown.enabled=true
+endpoints.shutdown.enabled=true
+#spring.profiles.active=portable
+spring.profiles.active=installer

vripper-ui/angular.json

@@ -27,9 +27,14 @@
               "src/assets"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "./node_modules/photoswipe/dist/photoswipe.css",
+              "./node_modules/photoswipe/dist/default-skin/default-skin.css"
             ],
-            "scripts": []
+            "scripts": [
+              "./node_modules/photoswipe/dist/photoswipe.min.js",
+              "./node_modules/photoswipe/dist/photoswipe-ui-default.min.js"
+            ]
           },
           "configurations": {
             "production": {
@@ -83,9 +88,14 @@
             "tsConfig": "src/tsconfig.spec.json",
             "karmaConfig": "src/karma.conf.js",
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "./node_modules/photoswipe/dist/photoswipe.css",
+              "./node_modules/photoswipe/dist/default-skin/default-skin.css"
+            ],
+            "scripts": [
+              "./node_modules/photoswipe/dist/photoswipe.min.js",
+              "./node_modules/photoswipe/dist/photoswipe-ui-default.min.js"
             ],
-            "scripts": [],
             "assets": [
               "src/favicon.ico",
               "src/assets"

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "1.5.1",
+  "version": "2.10.4",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -13,36 +13,37 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^7.2.3",
-    "@angular/cdk": "^7.3.1",
-    "@angular/common": "~7.2.0",
-    "@angular/compiler": "~7.2.0",
-    "@angular/core": "~7.2.0",
-    "@angular/flex-layout": "^7.0.0-beta.23",
-    "@angular/forms": "~7.2.0",
-    "@angular/material": "^7.3.0",
-    "@angular/platform-browser": "~7.2.0",
-    "@angular/platform-browser-dynamic": "~7.2.0",
-    "@angular/router": "~7.2.0",
-    "ag-grid-angular": "^21.0.1",
-    "ag-grid-community": "^21.0.1",
+    "@angular/animations": "^8.2.14",
+    "@angular/cdk": "^8.2.3",
+    "@angular/common": "~8.2.14",
+    "@angular/compiler": "~8.2.14",
+    "@angular/core": "~8.2.14",
+    "@angular/flex-layout": "^8.0.0-beta.27",
+    "@angular/forms": "~8.2.14",
+    "@angular/material": "^8.2.3",
+    "@angular/platform-browser": "~8.2.14",
+    "@angular/platform-browser-dynamic": "~8.2.14",
+    "@angular/router": "~8.2.14",
+    "ag-grid-angular": "^22.1.0",
+    "ag-grid-community": "^22.1.0",
     "core-js": "^2.5.4",
     "hammerjs": "^2.0.8",
-    "ngx-electron": "^2.1.1",
-    "rxjs": "~6.3.3",
+    "ngx-electron": "^2.2.0",
+    "photoswipe": "^4.1.3",
+    "rxjs": "~6.5.3",
     "tslib": "^1.9.0",
     "zone.js": "~0.8.26"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "^0.13.9",
-    "@angular/cli": "~7.2.2",
-    "@angular/compiler-cli": "^7.2.15",
-    "@angular/language-service": "~7.2.0",
+    "@angular-devkit/build-angular": "^0.803.23",
+    "@angular/cli": "~8.3.20",
+    "@angular/compiler-cli": "^8.2.14",
+    "@angular/language-service": "~8.2.14",
     "@types/jasmine": "~2.8.8",
     "@types/jasminewd2": "~2.0.3",
     "@types/node": "~8.9.4",
     "codelyzer": "~4.5.0",
-    "electron": "^6.0.4",
+    "electron": "^7.1.7",
     "jasmine-core": "~2.99.1",
     "jasmine-spec-reporter": "~4.2.1",
     "karma": "^4.2.0",
@@ -53,6 +54,6 @@
     "protractor": "~5.4.0",
     "ts-node": "~7.0.0",
     "tslint": "~5.11.0",
-    "typescript": "~3.2.2"
+    "typescript": "~3.5.0"
   }
 }

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.5.1</version>
+    <version>2.10.4</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.component.html

@@ -1,36 +1,25 @@
-<div fxLayout="column" *ngIf="connecting()" class="overlay loading" fxLayoutAlign="center center">
-  <mat-spinner></mat-spinner>
-  <h2>Connecting</h2>
-</div>
-<div fxLayout="column" *ngIf="noConnectionState()" class="overlay no-connection" fxLayoutAlign="center center">
-  <mat-icon class="overlay-icon">error</mat-icon>
-  <h2>Connection with the server is closed</h2>
-</div>
-<div [ngClass]="{'electron': electronService.isElectronApp}" id="app-container" fxLayout="column" fxLayoutGap="20px">
-  <div id="app-header" fxFlex="nogrow">
-    <mat-toolbar class="mat-elevation-z2" color="primary">
-      <mat-toolbar-row fxLayout="row" fxLayoutAlign="center center">
-        <span id="logo" fxFlex="0 0 48px"></span>
-        <span id="title">Viper Ripper</span>
-        <span fxFlex="grow"></span>
-        <span>
-          <p style="font-size: 16px">Logged in as: {{loggedUser.user == null || loggedUser.user === '' ? 'guest': loggedUser.user }}</p>
-        </span>
-        <div>
-          <button mat-icon-button [matMenuTriggerFor]="menu" aria-label="settings">
-            <mat-icon>more_vert</mat-icon>
-          </button>
-          <mat-menu #menu="matMenu">
-            <button (click)="openSettings()" mat-menu-item>
-              <mat-icon>settings</mat-icon>
-              <span>Settings</span>
-            </button>
-          </mat-menu>
-        </div>
-      </mat-toolbar-row>
-    </mat-toolbar>
+<ng-container *ngIf="appState | async as state">
+  <div *ngIf="state === 'CONNECTING'" class="overlay loading" fxLayout="column" fxLayoutAlign="center center">
+    <div class="sk-chase">
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+      <div class="sk-chase-dot"></div>
+    </div>
   </div>
-  <div id="app-body" fxFlex="grow">
-    <router-outlet></router-outlet>
+  <div *ngIf="state === 'DISCONNECTED'" class="overlay no-connection" fxLayout="column" fxLayoutAlign="center center">
+    <mat-icon class="overlay-icon">error</mat-icon>
+    <h2>Error connecting to daemon, try restarting the application</h2>
   </div>
-</div>
+  <div *ngIf="state === 'CONNECTED'" [ngClass]="{ electron: electron | async }" fxLayout="column" id="app-container">
+    <div fxFlex="nogrow" id="app-header">
+      <app-toolbar></app-toolbar>
+    </div>
+    <div fxFlex="grow" id="app-body" style="margin-bottom: 20px;">
+      <router-outlet></router-outlet>
+    </div>
+    <app-status-bar style="width: 100%"></app-status-bar>
+  </div>
+</ng-container>

vripper-ui/src/app/app.component.scss

@@ -1,25 +1,76 @@
-#logo {
-  height: 48px;
-  margin-right: 10px;
-  background-image: url('../assets/logo.png');
-  background-repeat: no-repeat;
-  background-size: 48px 48px;
-}
-
-#title {
-  user-select: none
-}
-
 .overlay-icon {
-  font-size: 128px;
-  width: 128px;
-  height: 128px;
+  font-size: 64px;
+  width: 64px;
+  height: 64px;
 }
 
-.no-connection {
-  color: #eb6060;
+.no-connection, .loading {
+  background-color: #2c3e50;
 }
 
-.electron {
-  margin-top: 30px;
+.overlay {
+  position: absolute;
+  display: block;
+  width: 100%;
+  height: 100%;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  z-index: 2000;
 }
+
+.sk-chase {
+  width: 60px;
+  height: 60px;
+  position: relative;
+  animation: sk-chase 2.5s infinite linear both;
+}
+
+.sk-chase-dot {
+  width: 100%;
+  height: 100%;
+  position: absolute;
+  left: 0;
+  top: 0; 
+  animation: sk-chase-dot 2.0s infinite ease-in-out both; 
+}
+
+.sk-chase-dot:before {
+  content: '';
+  display: block;
+  width: 25%;
+  height: 25%;
+  background-color: #fff;
+  border-radius: 100%;
+  animation: sk-chase-dot-before 2.0s infinite ease-in-out both; 
+}
+
+.sk-chase-dot:nth-child(1) { animation-delay: -1.1s; }
+.sk-chase-dot:nth-child(2) { animation-delay: -1.0s; }
+.sk-chase-dot:nth-child(3) { animation-delay: -0.9s; }
+.sk-chase-dot:nth-child(4) { animation-delay: -0.8s; }
+.sk-chase-dot:nth-child(5) { animation-delay: -0.7s; }
+.sk-chase-dot:nth-child(6) { animation-delay: -0.6s; }
+.sk-chase-dot:nth-child(1):before { animation-delay: -1.1s; }
+.sk-chase-dot:nth-child(2):before { animation-delay: -1.0s; }
+.sk-chase-dot:nth-child(3):before { animation-delay: -0.9s; }
+.sk-chase-dot:nth-child(4):before { animation-delay: -0.8s; }
+.sk-chase-dot:nth-child(5):before { animation-delay: -0.7s; }
+.sk-chase-dot:nth-child(6):before { animation-delay: -0.6s; }
+
+@keyframes sk-chase {
+  100% { transform: rotate(360deg); } 
+}
+
+@keyframes sk-chase-dot {
+  80%, 100% { transform: rotate(360deg); } 
+}
+
+@keyframes sk-chase-dot-before {
+  50% {
+    transform: scale(0.4); 
+  } 100%, 0% {
+    transform: scale(1.0); 
+  } 
+}

vripper-ui/src/app/app.component.scss-theme.scss

@@ -0,0 +1,11 @@
+@import '~@angular/material/theming';
+
+@mixin app-component-theme($theme) {
+
+  $background: map-get($theme, background);
+
+  button.add-button {
+
+    background-color: mat-color($background, background);
+  }
+}

vripper-ui/src/app/app.component.ts

@@ -1,91 +1,48 @@
+import { AppService } from './app.service';
 import { ClipboardService } from './clipboard.service';
 import { ElectronService } from 'ngx-electron';
-import { SettingsComponent } from './settings/settings.component';
-import { Component, OnInit, OnDestroy, NgZone } from '@angular/core';
+import { Component, OnDestroy, AfterViewInit, Renderer2, ChangeDetectionStrategy, NgZone } from '@angular/core';
 import { MatDialog } from '@angular/material';
-import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout';
-import { Observable, Subscription } from 'rxjs';
+import { Subscription, BehaviorSubject, Subject, merge } from 'rxjs';
 import { WsConnectionService, WSState } from './ws-connection.service';
-import { LoggedUser } from './common/logged-user.model';
-import { WsHandler } from './ws-handler';
-import { CMD } from './common/cmd.enum';
-import { WSMessage } from './common/ws-message.model';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
-  styleUrls: ['./app.component.scss']
+  styleUrls: ['./app.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
-export class AppComponent implements OnInit, OnDestroy {
+export class AppComponent implements OnDestroy, AfterViewInit {
   constructor(
-    public dialog: MatDialog,
-    private breakpointObserver: BreakpointObserver,
+    private dialog: MatDialog,
     private ws: WsConnectionService,
     public electronService: ElectronService,
     private clipboardService: ClipboardService,
-    private ngZone: NgZone,
+    private appService: AppService,
+    private renderer: Renderer2,
+    private ngZone: NgZone
   ) {
-    this.websocketHandlerPromise = this.ws.getConnection();
+    this.electron = new BehaviorSubject(electronService.isElectronApp);
   }
 
-  subscriptions: Subscription[] = [];
-  websocketHandlerPromise: Promise<WsHandler>;
-  currentState: WSState;
+  private subscriptions: Subscription[] = [];
+  appState: Subject<string> = new BehaviorSubject('CONNECTING');
+  electron: Subject<boolean>;
 
-  loggedUser: LoggedUser = new LoggedUser(null);
-
-  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
-
-  openSettings(): void {
-    const dialogRef = this.dialog.open(SettingsComponent, {
-      width: '70%',
-      height: '70%',
-      maxWidth: '100vw',
-      maxHeight: '100vh'
-    });
-
-    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
-      if (result.matches) {
-        dialogRef.updateSize('100%', '100%');
-      } else {
-        dialogRef.updateSize('70%', '70%');
-      }
-    });
-
-    dialogRef.afterClosed().subscribe(result => {
-      smallDialogSubscription.unsubscribe();
-    });
-  }
-
-  noConnectionState(): boolean {
-    return this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR;
-  }
-
-  connecting(): boolean {
-    return this.currentState === WSState.INIT || this.currentState === WSState.CONNECTING;
-  }
-
-  ngOnInit() {
-    this.websocketHandlerPromise.then((handler: WsHandler) => {
-      console.log('Connecting to user stream');
-      this.subscriptions.push(
-        handler.subscribeForUser((e: LoggedUser[]) => {
-          this.ngZone.run(() => {
-            this.loggedUser = e[0];
-          });
-        })
-      );
-      handler.send(new WSMessage(CMD.USER_SUB.toString()));
-    });
-    this.currentState = WSState.INIT;
-    this.ws.state.subscribe(e => {
-      this.currentState = e;
-      if (this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR) {
-        this.dialog.closeAll();
-      } else if (this.currentState === WSState.OPEN) {
+  ngAfterViewInit() {
+    this.appService.renderer = this.renderer;
+    this.subscriptions.push(this.ws.state.subscribe(wsState => {
+      if (wsState === WSState.CLOSE || wsState === WSState.ERROR) {
+        setTimeout(() => this.ngZone.run(() => {
+          this.dialog.closeAll();
+          this.appState.next('DISCONNECTED');
+        }), 500);
+      } else if (wsState === WSState.OPEN) {
         this.clipboardService.init();
+        merge(this.appService.loadTheme(), this.appService.loadSettings())
+        .subscribe(() => this.ngZone.run(() => this.appState.next('CONNECTED')));
       }
-    });
+    }));
   }
 
   ngOnDestroy() {

vripper-ui/src/app/app.module.ts

@@ -1,3 +1,7 @@
+import { LinkCollectorService } from './link-collector.service';
+import { ContextMenuService } from './ctxt-menu.service';
+import { PostContextMenuComponent } from './posts/post-context-menu.component';
+import { PostsDataService } from './posts-data.service';
 import { AppPreviewDirective } from './common/preview-tooltip.directive';
 import { WsConnectionService } from './ws-connection.service';
 import { AppService } from './app.service';
@@ -14,7 +18,6 @@ import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { AgGridModule } from 'ag-grid-angular';
 import { PostProgressRendererComponent } from './posts/post-progress.renderer.component';
-import { MenuRendererComponent } from './posts/menu.renderer.component';
 import { PostDetailsProgressRendererComponent } from './post-detail/post-details-progress.component';
 import { LoginComponent } from './login/login.component';
 import { XhrInterceptorService } from './xhr-interceptor.service';
@@ -29,6 +32,15 @@ import { MultiPostComponent } from './multi-post/multi-post.component';
 import { OverlayModule } from '@angular/cdk/overlay';
 import { AppPreviewComponent } from './common/preview-tooltip.component';
 import { UrlRendererComponent } from './multi-post/url-renderer.component';
+import { FilterComponent } from './filter/filter.component';
+import { ScanComponent } from './scan/scan.component';
+import { StatusBarComponent } from './status-bar/status-bar.component';
+import { SelectionService } from './selection-service';
+import { ToolbarComponent } from './toolbar/toolbar.component';
+import { GrabQueueComponent } from './grab-queue/grab-queue.component';
+import { UrlGrabRendererComponent } from './grab-queue/url-renderer.component';
+import { GalleryComponent } from './gallery/gallery.component';
+import { PhotoSwipeComponent } from './photo-swipe/photo-swipe.component';
 
 @NgModule({
   declarations: [
@@ -36,7 +48,6 @@ import { UrlRendererComponent } from './multi-post/url-renderer.component';
     PostsComponent,
     PostDetailComponent,
     PostProgressRendererComponent,
-    MenuRendererComponent,
     PostDetailsProgressRendererComponent,
     LoginComponent,
     HomeComponent,
@@ -45,9 +56,27 @@ import { UrlRendererComponent } from './multi-post/url-renderer.component';
     MultiPostComponent,
     AppPreviewComponent,
     AppPreviewDirective,
-    UrlRendererComponent
+    UrlRendererComponent,
+    UrlGrabRendererComponent,
+    FilterComponent,
+    ScanComponent,
+    StatusBarComponent,
+    ToolbarComponent,
+    GrabQueueComponent,
+    PostContextMenuComponent,
+    GalleryComponent,
+    PhotoSwipeComponent
+  ],
+  entryComponents: [
+    PostDetailComponent,
+    SettingsComponent,
+    ConfirmDialogComponent,
+    AppPreviewComponent,
+    ScanComponent,
+    MultiPostComponent,
+    PostContextMenuComponent,
+    GalleryComponent
   ],
-  entryComponents: [PostDetailComponent, SettingsComponent, ConfirmDialogComponent, MultiPostComponent, AppPreviewComponent],
   imports: [
     BrowserAnimationsModule,
     FormsModule,
@@ -59,9 +88,9 @@ import { UrlRendererComponent } from './multi-post/url-renderer.component';
     ReactiveFormsModule,
     AgGridModule.withComponents([
       PostProgressRendererComponent,
-      MenuRendererComponent,
       PostDetailsProgressRendererComponent,
-      UrlRendererComponent
+      UrlRendererComponent,
+      UrlGrabRendererComponent
     ]),
     OverlayModule
   ],
@@ -70,7 +99,11 @@ import { UrlRendererComponent } from './multi-post/url-renderer.component';
     WsConnectionService,
     { provide: HTTP_INTERCEPTORS, useClass: XhrInterceptorService, multi: true },
     ServerService,
-    SharedService
+    SharedService,
+    SelectionService,
+    PostsDataService,
+    ContextMenuService,
+    LinkCollectorService
   ],
   bootstrap: [AppComponent]
 })

vripper-ui/src/app/app.service.ts

@@ -1,4 +1,101 @@
-import { Injectable } from '@angular/core';
+import { Settings } from './common/settings.model';
+import { environment } from 'src/environments/environment';
+import { MatDialog } from '@angular/material';
+import { HttpClient } from '@angular/common/http';
+import { Injectable, Renderer2 } from '@angular/core';
+import { ServerService } from './server-service';
+import { tap } from 'rxjs/operators';
+import { ScanComponent } from './scan/scan.component';
+import { Observable, Subject } from 'rxjs';
+import { BreakpointState, Breakpoints, BreakpointObserver } from '@angular/cdk/layout';
+import { Title } from '@angular/platform-browser';
 
 @Injectable()
-export class AppService {}
+export class AppService {
+  constructor(
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    public dialog: MatDialog,
+    private breakpointObserver: BreakpointObserver,
+    private titleService: Title
+  ) {
+    this.titleService.setTitle('VRipper ' + environment.version);
+  }
+
+  private _darkTheme = false;
+  private _settings: Settings;
+  private _renderer: Renderer2;
+
+  get settings(): Settings {
+    return { ...this._settings };
+  }
+
+  get darkTheme(): boolean {
+    return this._darkTheme;
+  }
+
+  set renderer(renderer: Renderer2) {
+    this._renderer = renderer;
+  }
+
+  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
+
+  updateTheme(darkTheme: boolean) {
+    this._darkTheme = darkTheme;
+    if (this._darkTheme) {
+      this._renderer.addClass(document.body, 'dark-theme');
+      this._renderer.removeClass(document.body, 'light-theme');
+    } else {
+      this._renderer.addClass(document.body, 'light-theme');
+      this._renderer.removeClass(document.body, 'dark-theme');
+    }
+    this.httpClient
+      .post<Theme>(this.serverService.baseUrl + '/settings/theme', {
+        darkTheme: this._darkTheme
+      })
+      .subscribe();
+  }
+
+  updateSettings(settings: Settings) {
+    this._settings = settings;
+  }
+
+  loadTheme() {
+    return this.httpClient
+      .get<Theme>(this.serverService.baseUrl + '/settings/theme')
+      .pipe(tap(theme => this.updateTheme(theme.darkTheme)));
+  }
+
+  loadSettings() {
+    return this.httpClient
+      .get<Settings>(this.serverService.baseUrl + '/settings')
+      .pipe(tap(settings => this.updateSettings(settings)));
+  }
+
+  scan(url?: string) {
+
+    const scanDialog = this.dialog.open(ScanComponent, {
+      width: '70%',
+      height: '70%',
+      maxWidth: '100vw',
+      maxHeight: '100vh',
+      data: {url: url}
+    });
+
+    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
+      if (result.matches) {
+        scanDialog.updateSize('100%', '100%');
+      } else {
+        scanDialog.updateSize('70%', '70%');
+      }
+    });
+
+    scanDialog.afterClosed().subscribe(() => {
+      smallDialogSubscription.unsubscribe();
+    });
+  }
+}
+
+export interface Theme {
+  darkTheme: boolean;
+}

vripper-ui/src/app/clipboard.service.ts

@@ -1,6 +1,6 @@
 import { Subject, Observable } from 'rxjs';
 import { ElectronService } from 'ngx-electron';
-import { Injectable, OnInit } from '@angular/core';
+import { Injectable } from '@angular/core';
 import { Clipboard } from 'electron';
 import { HttpClient } from '@angular/common/http';
 import { Settings } from './common/settings.model';

vripper-ui/src/app/common/cmd.enum.ts

@@ -5,10 +5,12 @@ export enum CMD {
     SPEED_SUB = 'SPEED_SUB',
     USER_SUB = 'USER_SUB',
     THREAD_PARSING_SUB = 'THREAD_PARSING_SUB',
+    GRAB_QUEUE_SUB = 'GRAB_QUEUE_SUB',
     POSTS_UNSUB = 'POSTS_UNSUB',
     POST_DETAILS_UNSUB = 'POST_DETAILS_UNSUB',
     GLOBAL_STATE_UNSUB = 'GLOBAL_STATE_UNSUB',
     SPEED_UNSUB = 'SPEED_UNSUB',
     USER_UNSUB = 'USER_UNSUB',
-    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB'
+    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB',
+    GRAB_QUEUE_UNSUB = 'GRAB_QUEUE_UNSUB'
 }

vripper-ui/src/app/common/preview-tooltip.component.ts

@@ -4,11 +4,10 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
 @Component({
   selector: 'app-preview',
   template: `
-  <div class="previews" style="display: inline-block; white-space: nowrap;">
+  <div class="previews" style="display: inline-block; white-space: nowrap; height: 200px; max-width: 400px">
     <ng-container *ngFor="let link of links">
       <img
         [@simpleFadeAnimation]="'in'"
-        class="mat-elevation-z8"
         [src]="link"
       />
     </ng-container>
@@ -18,9 +17,8 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
     `
       img {
         max-height: 200px;
-        max-width: 200px;
+        max-width: 400px;
         margin-left: 5px;
-        border: 1px solid #900000;
       }
     `
   ],

vripper-ui/src/app/common/preview-tooltip.directive.ts

@@ -1,30 +1,48 @@
 import { AppPreviewComponent } from './preview-tooltip.component';
-import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone } from '@angular/core';
+import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone, OnDestroy } from '@angular/core';
 import { OverlayRef, Overlay, OverlayPositionBuilder } from '@angular/cdk/overlay';
 import { ComponentPortal } from '@angular/cdk/portal';
 
 @Directive({ selector: '[appPreview]' })
-export class AppPreviewDirective implements OnInit {
+export class AppPreviewDirective implements OnInit, OnDestroy {
   private overlayRef: OverlayRef;
   tooltipPortal = new ComponentPortal(AppPreviewComponent);
 
   @Input() appPreview: string[];
 
-  constructor(private overlayPositionBuilder: OverlayPositionBuilder,
+  constructor(
+    private overlayPositionBuilder: OverlayPositionBuilder,
     private elementRef: ElementRef,
     private overlay: Overlay,
-    private ngZone: NgZone) {}
+    private ngZone: NgZone
+  ) {}
 
   ngOnInit() {
     const positionStrategy = this.overlayPositionBuilder
-    .flexibleConnectedTo(this.elementRef)
-    .withPositions([{
-      originX: 'center',
-      originY: 'top',
-      overlayX: 'center',
-      overlayY: 'bottom',
-    }]);
-    this.overlayRef = this.overlay.create({ positionStrategy });
+      .flexibleConnectedTo(this.elementRef)
+      .withPush(true)
+      .withGrowAfterOpen(true)
+      .withPositions([
+        {
+          originX: 'center',
+          originY: 'bottom',
+          overlayX: 'center',
+          overlayY: 'top'
+        },
+        {
+          originX: 'center',
+          originY: 'top',
+          overlayX: 'center',
+          overlayY: 'bottom'
+        }
+      ]);
+    this.overlayRef = this.overlay.create({ positionStrategy, scrollStrategy: this.overlay.scrollStrategies.close() });
+  }
+
+  ngOnDestroy() {
+    this.ngZone.run(() => {
+      this.overlayRef.dispose();
+    });
   }
 
   @HostListener('mouseenter')

vripper-ui/src/app/common/settings.model.ts

@@ -1,10 +1,13 @@
 export interface Settings {
     downloadPath: string;
     maxThreads: number;
+    maxTotalThreads: number;
     autoStart: boolean;
     vLogin: boolean;
     vUsername: string;
     vPassword: string;
     vThanks: boolean;
     desktopClipboard: boolean;
-  }
+    viewPhotos: boolean;
+    notification: boolean;
+  }

vripper-ui/src/app/common/vr-post-parse.model.ts

@@ -6,10 +6,7 @@ export class VRPostParse {
     public title: string,
     public imageCount: number,
     public url: string,
-    public previews: string[]
+    public previews: string[],
+    public hosts: string
   ) {}
 }
-
-export class VRThreadParseState {
-  constructor(public threadId: string, public state: string) {}
-}

vripper-ui/src/app/ctxt-menu.service.ts

@@ -0,0 +1,68 @@
+import { PostState } from './posts/post-state.model';
+import { Injectable, ComponentRef } from '@angular/core';
+import { filter, take } from 'rxjs/operators';
+import { fromEvent, Subscription } from 'rxjs';
+import { OverlayPositionBuilder, Overlay, OverlayRef } from '@angular/cdk/overlay';
+import { ComponentPortal } from '@angular/cdk/portal';
+import { PostContextMenuComponent } from './posts/post-context-menu.component';
+
+@Injectable()
+export class ContextMenuService {
+  constructor(private overlayPositionBuilder: OverlayPositionBuilder, private overlay: Overlay) {}
+
+  postCtxtMenuPortal = new ComponentPortal(PostContextMenuComponent);
+  private postCtxtMenuOverlayRef: OverlayRef;
+  sub: Subscription;
+
+  openPostCtxtMenu(mouseEvent: MouseEvent, postState: PostState) {
+    mouseEvent.preventDefault();
+    this.closePostCtxtMenu();
+    const positionStrategy = this.overlayPositionBuilder
+      .flexibleConnectedTo({ x: mouseEvent.x, y: mouseEvent.y })
+      .withPush(true)
+      .withGrowAfterOpen(true)
+      .withPositions([
+        {
+          originX: 'start',
+          originY: 'bottom',
+          overlayX: 'start',
+          overlayY: 'top'
+        },
+        {
+          originX: 'start',
+          originY: 'top',
+          overlayX: 'start',
+          overlayY: 'bottom'
+        }
+      ]);
+    this.postCtxtMenuOverlayRef = this.overlay.create({
+      positionStrategy
+    });
+    const ref: ComponentRef<PostContextMenuComponent> = this.postCtxtMenuOverlayRef.attach(this.postCtxtMenuPortal);
+    ref.instance.postState = postState;
+    ref.instance.contextMenuService = this;
+    this.sub = fromEvent<MouseEvent>(document, 'click')
+      .pipe(
+        filter(event => {
+          const clickTarget = event.target as HTMLElement;
+          return !!this.postCtxtMenuOverlayRef && !this.postCtxtMenuOverlayRef.overlayElement.contains(clickTarget);
+        }),
+        take(1)
+      )
+      .subscribe(() => this.closePostCtxtMenu());
+  }
+
+  closePostCtxtMenu() {
+    if (this.sub != null) {
+      this.sub.unsubscribe();
+    }
+    if (this.postCtxtMenuOverlayRef) {
+      this.postCtxtMenuOverlayRef.dispose();
+      this.postCtxtMenuOverlayRef = null;
+    }
+  }
+}
+
+export interface Theme {
+  darkTheme: boolean;
+}

vripper-ui/src/app/filter/filter.component.html

@@ -0,0 +1,3 @@
+<mat-chip-list>
+  <mat-chip (click)="select(c)" *ngFor="let c of criteria" [selected]="c.selected" color="accent">{{c.value}}</mat-chip>
+</mat-chip-list>

vripper-ui/src/app/filter/filter.component.ts

@@ -0,0 +1,26 @@
+import { Component, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'app-filter',
+  templateUrl: './filter.component.html',
+  styleUrls: ['./filter.component.scss']
+})
+export class FilterComponent implements OnInit {
+
+  constructor() { }
+
+  criteria = [
+    {value: 'All', selected: true},
+    {value: 'Downloading', selected: false},
+    {value: 'Completed', selected: false},
+    {value: 'Error', selected: false},
+  ];
+
+  select(chip: {value: string, selected: boolean}) {
+    this.criteria.forEach(e => e.selected = false);
+    chip.selected = true;
+  }
+
+  ngOnInit() {
+  }
+}