v2.3.1

Fix Post Status when autostart option is selected
Add stop all button

.gitignore

@@ -2,4 +2,5 @@
 /**/*/node_modules
 /**/*/target
 /**/*/build-dir
+/**/*/java-runtime
 .idea

CHANGELOG.md

@@ -1,5 +1,157 @@
 # Changelog
 
+## [2.3.1] - 2019-11-22
+### Changed
+- Fix connexion reset issue
+- Fix Post Status when autostart option is selected
+- Add stop all button
+
+## [2.3.0] - 2019-11-21
+### Changed
+- UI enhancements and overhaul
+- Stability fixes with HTTP connections
+- MacOS official support
+- Use local sockjs and material icons font
+
+## [2.2.5] - 2019-11-09
+### Changed
+- Exclude thumbs download for imagezilla and pixhost
+- Change retry on fail logic (may help avoid having download errors)
+
+## [2.2.4] - 2019-10-05
+### Added
+- AppImage target for linux
+
+## [2.2.3] - 2019-10-01
+### Changed
+- Strip libjvm.so for debian
+
+## [2.2.2] - 2019-10-01
+### Changed
+- Update product name
+
+## [2.2.1] - 2019-10-01
+### Changed
+- Update dependency list for deb package
+
+## [2.2.0] - 2019-10-01
+### Added
+- Embedd JRE within the app
+### Changed
+- Fix 'Start' on pending items
+- Upgrade to Java 11
+
+## [2.1.0] - 2019-09-30
+### Added
+- Add a create a subfolder per thread option
+
+## [2.0.4] - 2019-09-27
+### Changed
+- Fix some issues with ImxHost
+
+## [2.0.3] - 2019-09-23
+### Changed
+- Update deb dependencies
+
+## [2.0.2] - 2019-09-23
+### Changed
+- Add portable mode
+
+## [2.0.1] - 2019-09-22
+### Changed
+- Change app name
+
+## [2.0.0] - 2019-09-22
+### Added
+- Add sub folder option
+- Add clear finished option
+### Changed
+- UI enhancements
+- Bug fixes
+- Change data and log paths
+
+## [1.6.5] - 2019-09-18
+### Changed
+- electron update
+
+## [1.6.4] - 2019-09-18
+### Changed
+- Change menu item name
+
+## [1.6.3] - 2019-09-18
+### Changed
+- Better multithread management
+- Fix partial state UI
+
+## [1.6.2] - 2019-09-17
+### Changed
+- Rework UI colors
+- Fix scan input bug
+
+## [1.6.1] - 2019-09-16
+### Changed
+- Rework UI
+- Fix clipboard monitoring
+
+## [1.6.0] - 2019-09-15
+### Changed
+- Rework the UI
+- Add a dark theme
+- Fix the force ordering bug
+- Other bug fixes
+
+## [1.5.2] - 2019-09-01
+### Added
+- imx.to host fix
+
+## [1.5.1] - 2019-08-24
+### Added
+- Bug fixes
+
+## [1.5.0] - 2019-08-24
+### Added
+- Display logged in user
+- Add post details when clicking on main list items (Status, Post URL, Host)
+- Add open download location on main list items menu
+- Add support for multi-post threads (Show a menu to select posts to download)
+- Add link to image from details list
+
+## [1.4.10] - 2019-08-15
+### Changed
+- Fix a rename issue on windows
+
+## [1.4.9] - 2019-08-13
+### Changed
+- Update Built-By to be generic
+
+## [1.4.8] - 2019-08-13
+### Changed
+- Add option to force image ordering
+
+## [1.4.7] - 2019-08-13
+### Changed
+- Fixed a bug in image filename extensions 
+
+## [1.4.6] - 2019-08-06
+### Changed
+- Fixed a bug when creating image file names
+- Remove unnecessary logs
+- Fix data.json init 
+
+## [1.4.5] - 2019-08-03
+### Changed
+- Add min width and height for electron app
+### Added
+- Add download speed to status bar
+
+## [1.4.4] - 2019-08-01
+### Changed
+- Fix names for ImgboxHost
+- Enhance UI
+- Minor fixes
+### Added
+- Add start and stop button
+
 ## [1.4.3] - 2019-07-14
 ### Changed
 - Update some messages

README.md

@@ -0,0 +1,26 @@
+# Viper Ripper!
+
+This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
+The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
+
+## How to build
+You need a recent version of maven 3.6.1+.
+
+The application support 2 build modes:
+ - Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
+ - Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
+
+Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
+
+To build the server app:
+
+    mvn clean install
+To build the desktop app:
+
+    mvn clean install -Pelectron
+
+Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
+
+*Partially cross platform:
+Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
+The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.3</version>
+    <version>2.3.1</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>
@@ -15,5 +15,6 @@
     <modules>
         <module>vripper-server</module>
         <module>vripper-ui</module>
+        <module>vripper-electron</module>
     </modules>
 </project>

vripper-electron/main.js

@@ -4,40 +4,36 @@ const url = require("url");
 const getPort = require("get-port");
 const { spawn } = require("child_process");
 const { ipcMain } = require("electron");
-const commandExists = require("command-exists").sync;
 const { dialog } = require("electron");
+const axios = require('axios');
 const appDir = process.env.APPDIR;
 
-console.log('App path', path.join(app.getPath('exe'), "../bin/vripper-server.jar"));
-console.log('App image path', appDir);
-
 let win;
 let vripperServer;
+let serverPort;
+
+const maxTerminationAttemps = 5;
+let terminationAttemps = 0;
+let terminationInteval;
+let terminated = false;
 
 process.on("uncaughtException", err => {
   dialog.showErrorBox(err.message, err.stack);
   process.exit(1);
 });
 
-if (!commandExists("java")) {
-  dialog.showErrorBox(
-    "Java command is missing",
-    "Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
-  );
-  process.exit(1);
-}
-
 function createWindow() {
   let icon;
   if(process.platform === "win32") {
-    icon = path.join(__dirname, `icon.ico`);
+    icon = __dirname + '/icon.ico';
   } else {
-    icon = path.join(__dirname, `icon.icns`);
+    icon = __dirname + '/icon.png';
   }
   win = new BrowserWindow({
     width: 1024,
-    height: 768,
-    frame: false,
+    height: 800,
+    minWidth: 800,
+    minHeight: 600,
     webPreferences: {
       nodeIntegration: true
     },
@@ -45,6 +41,8 @@ function createWindow() {
   });
 
   win.removeMenu();
+  win.setMenu(null);
+  win.setMenuBarVisibility(false);
 
   win.loadURL(
     url.format({
@@ -62,15 +60,44 @@ function createWindow() {
 }
 
 getPort().then(port => {
+  serverPort = port;
   ipcMain.on("get-port", event => {
     event.reply("port", port);
   });
-  vripperServer = spawn("java", [
+  let javaBinPath;
+  if(appDir !== undefined) {
+    javaBinPath = path.join(appDir, "java-runtime/bin/java");
+  } else {
+    if(process.platform === 'darwin') {
+      javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+    } else {
+      javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+    }
+  }
+  let jarPath;
+  if(appDir !== undefined) {
+    jarPath = path.join(appDir, "bin/vripper-server.jar");
+  } else {
+    if(process.platform === 'darwin') {
+      jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+    } else {
+      jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+    }
+  }
+  vripperServer = spawn(javaBinPath, [
+    "-Xms256m",
+    "-Xmx1024m",
     "-Dvripper.server.port=" + port,
     "-jar",
-    appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
-    path.join(app.getPath('exe'), "../bin/vripper-server.jar")
-  ], {stdio: 'ignore'});
+    jarPath
+  ], {
+    stdio: 'ignore'
+  });
+  vripperServer.on('exit', (code, signal) => {
+    console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
+    terminated = true;
+    app.quit();
+  });
 });
 
 const gotTheLock = app.requestSingleInstanceLock();
@@ -89,10 +116,30 @@ if (!gotTheLock) {
 
   app.on("window-all-closed", () => {
     if (process.platform !== "darwin") {
-      if(vripperServer != null) {
-        vripperServer.kill("SIGTERM");
-      }
-      app.quit();
+      axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
+        headers: { 'content-type': 'application/json' },
+      }).then((response) => {
+        terminationInteval=  setInterval(() => {
+          terminationAttemps++;
+          if(terminated) {
+            console.log('viper server terminated');
+            clearInterval(terminationInteval);
+            app.quit();
+          } else if(terminationAttemps > maxTerminationAttemps) {
+            console.log('viper server is not terminated');
+            console.log('Proceed to kill');
+            vripperServer.kill('SIGKILL');
+            clearInterval(terminationInteval);
+            app.quit();
+          }
+        }, 1000);
+      })
+      .catch((error) => {
+        // Terminate immediately
+        vripperServer.kill('SIGKILL');
+        terminated = true;
+        app.quit();
+      });
     }
   });
 

vripper-electron/package.json

@@ -1,12 +1,14 @@
 {
   "name": "vripper-electron",
-  "version": "1.4.3",
-  "description": "",
+  "version": "2.3.1",
+  "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
-  "author": "",
+  "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
+  "homepage": "https://github.com/death-claw/vripper-project",
   "license": "ISC",
   "build": {
     "appId": "tn.mnlr.vripper",
+    "productName": "vripper",
     "files": [
       "**/*",
       "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -27,30 +29,53 @@
     ],
     "extraFiles": [
       {
-        "from": "../vripper-server/target/vripper-server-${version}.jar",
+        "from": "../vripper-server/target/vripper-server-${version}-electron.jar",
         "to": "bin/vripper-server.jar"
+      },
+      {
+        "from": "java-runtime",
+        "to": "java-runtime"
       }
     ],
     "win": {
-      "target": "nsis"
+      "icon": "icon.png",
+      "target": [
+        "nsis"
+      ]
+    },
+    "nsis": {
+      "createDesktopShortcut": "always",
+      "oneClick": false,
+      "perMachine": false
     },
     "linux": {
+      "synopsis": "vipergirls.to ripper",
+      "category": "Utility",
+      "packageCategory": "Utility",
+      "icon": "icons",
       "target": [
         "AppImage"
       ]
+    },
+    "mac": {
+      "category": "public.app-category.utilities",
+      "target": [
+        "dmg"
+      ],
+      "icon": "icon.png"
     }
   },
   "scripts": {
     "start": "electron .",
-    "dist": "node pre-build.js && build"
+    "dist": "node pre-build.js && electron-builder"
   },
   "devDependencies": {
-    "electron": "^5.0.5",
-    "electron-builder": "^20.44.4"
+    "electron": "^6.0.9",
+    "electron-builder": "^21.2.0"
   },
   "dependencies": {
+    "axios": "^0.19.0",
     "cheerio": "^1.0.0-rc.3",
-    "command-exists": "^1.2.8",
     "copy-dir": "^1.1.0",
     "custom-electron-titlebar": "^3.0.10",
     "electron-context-menu": "^0.13.0",

vripper-electron/pom.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>tn.mnlr</groupId>
+    <artifactId>vripper</artifactId>
+    <version>2.3.1</version>
+  </parent>
+  <artifactId>vripper-electron</artifactId>
+  <name>vripper-electron</name>
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-jar-plugin</artifactId>
+        <configuration>
+          <classifier>${buildClassifier}</classifier>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+  <profiles>
+    <profile>
+      <id>electron</id>
+      <properties>
+        <buildClassifier>electron</buildClassifier>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>tn.mnlr</groupId>
+          <artifactId>vripper-server</artifactId>
+          <version>${project.version}</version>
+          <scope>runtime</scope>
+          <classifier>${buildClassifier}</classifier>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.github.eirslett</groupId>
+            <artifactId>frontend-maven-plugin</artifactId>
+            <version>1.3</version>
+            <configuration>
+              <nodeVersion>v12.2.0</nodeVersion>
+              <npmVersion>6.9.0</npmVersion>
+              <workingDirectory>build-dir</workingDirectory>
+            </configuration>
+            <executions>
+              <execution>
+                <id>install node and npm</id>
+                <goals>
+                  <goal>install-node-and-npm</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>npm install</id>
+                <goals>
+                  <goal>npm</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>npm run build</id>
+                <goals>
+                  <goal>npm</goal>
+                </goals>
+                <configuration>
+                  <arguments>run dist</arguments>
+                </configuration>
+                <phase>generate-resources</phase>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>

vripper-electron/pre-build.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
 const fs = require('fs');
 const rimraf = require("rimraf");
 const copydir = require("copy-dir");
-
+const { execSync } = require("child_process");
 
 rimraf.sync('./build/vripper-ui');
 copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
 
 fs.writeFileSync('./build/vripper-ui/index.html', $.html());
 
+console.log('Building runtime environment');
+rimraf.sync('java-runtime');
+execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
+if(process.platform === 'linux') {
+    console.log('Stripping libjvm.so');
+    execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
+}

vripper-electron/renderer.js

@@ -1,12 +1,3 @@
 console.log("renderer intialized");
-const customTitlebar = require("custom-electron-titlebar");
-const electron = require("electron").remote;
 const contextMenu = require("electron-context-menu");
-
-contextMenu({});
-const menu = new electron.Menu();
-const titlebar = new customTitlebar.Titlebar({
-  backgroundColor: customTitlebar.Color.fromHex("#3f51b5"),
-  menu: menu
-});
-titlebar.updateTitle("Viper Ripper");
+contextMenu({});

vripper-server/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>1.4.3</version>
+        <version>2.3.1</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
     <description>vripper-server</description>
 
     <properties>
-        <java.version>1.8</java.version>
+        <java.version>11</java.version>
     </properties>
 
     <dependencies>
@@ -28,7 +28,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-websocket</artifactId>
         </dependency>
-
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
@@ -69,33 +72,29 @@
             <artifactId>vripper-ui</artifactId>
             <version>${project.version}</version>
             <scope>runtime</scope>
+            <classifier>${buildClassifier}</classifier>
         </dependency>
     </dependencies>
 
     <build>
-        <finalName>${project.artifactId}-${project.version}</finalName>
         <plugins>
+            <plugin>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <classifier>${buildClassifier}</classifier>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <artifactId>maven-resources-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>copy-resources</id>
-                        <phase>validate</phase>
+                        <id>repackage</id>
                         <goals>
-                            <goal>copy-resources</goal>
+                            <goal>repackage</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.build.directory}/classes/static</outputDirectory>
-                            <resources>
-                                <resource>
-                                    <directory>${project.parent.basedir}/vripper-ui/dist/vripper-ui/
-                                    </directory>
-                                </resource>
-                            </resources>
+                            <classifier>${buildClassifier}</classifier>
                         </configuration>
                     </execution>
                 </executions>
@@ -130,6 +129,7 @@
                     <archive>
                         <manifestEntries>
                             <Build-Time>${maven.build.timestamp}</Build-Time>
+                            <Built-By>${user.name}</Built-By>
                         </manifestEntries>
                         <manifestSections>
                             <manifestSection>
@@ -149,4 +149,47 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>web</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <buildClassifier>web</buildClassifier>
+            </properties>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-resources-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>copy-resources</id>
+                                <phase>validate</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}/classes/static</outputDirectory>
+                                    <resources>
+                                        <resource>
+                                            <directory>${project.parent.basedir}/vripper-ui/dist/vripper-ui/
+                                            </directory>
+                                        </resource>
+                                    </resources>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>electron</id>
+            <properties>
+                <buildClassifier>electron</buildClassifier>
+            </properties>
+        </profile>
+    </profiles>
 </project>

vripper-server/src/main/java/tn/mnlr/vripper/SpringContext.java

@@ -7,7 +7,6 @@ import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.stereotype.Component;
-import tn.mnlr.vripper.services.PersistenceService;
 
 import java.util.Map;
 
@@ -18,20 +17,11 @@ public class SpringContext implements ApplicationContextAware {
 
     private static ConfigurableApplicationContext context;
 
-    private PersistenceService persistenceService;
-
-    /**
-     * Returns the Spring managed bean instance of the given class type (if it exists).
-     * Returns null otherwise.
-     *
-     * @param beanClass
-     * @return
-     */
-    public static <T extends Object> T getBean(Class<T> beanClass) {
+    public static <T> T getBean(Class<T> beanClass) {
         return context.getBean(beanClass);
     }
 
-    public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
+    public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
         return context.getBeansOfType(beanClass);
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java

@@ -1,8 +1,10 @@
 package tn.mnlr.vripper;
 
+import lombok.Getter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -12,17 +14,18 @@ import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.PersistenceService;
 import tn.mnlr.vripper.services.VipergirlsAuthService;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @SpringBootApplication
 public class VripperApplication {
 
     private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
-
-    public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json";
+    public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
     public static void main(String[] args) {
-
         try {
             SpringApplication.run(VripperApplication.class, args);
         } catch (Exception e) {
@@ -42,6 +45,17 @@ public class VripperApplication {
         @Autowired
         private AppSettingsService appSettingsService;
 
+        @Value("${base.dir}")
+        private String baseDir;
+
+        @Getter
+        private String dataPath;
+
+        @PostConstruct
+        public void init() {
+            dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
+        }
+
         @Override
         public void run(String... args) {
 
@@ -61,5 +75,3 @@ public class VripperApplication {
         }
     }
 }
-
-

vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java

@@ -5,19 +5,22 @@ import io.reactivex.processors.BehaviorProcessor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.Objects;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 @Getter
 @NoArgsConstructor
 public class Image {
 
-    @Setter
+    private static final Logger logger = LoggerFactory.getLogger(Image.class);
+
     private AppStateService appStateService;
+
     private Disposable subscription;
 
     private final String type = "img";
@@ -30,6 +33,8 @@ public class Image {
 
     private String url;
 
+    private int index;
+
     private AtomicLong current = new AtomicLong(0);
     private Status status;
     private BehaviorProcessor<Image> imageStateProcessor;
@@ -37,17 +42,20 @@ public class Image {
     @Setter
     private long total = 0;
 
-    public Image(String url, String postId, String postName, Host host, AppStateService appStateService) {
+    public Image(String url, String postId, String postName, Host host, int index) {
         this.url = url;
         this.postId = postId;
         this.postName = postName;
         this.host = host;
-        this.appStateService = appStateService;
-        status = Status.PENDING;
+        this.index = index;
+        status = Status.STOPPED;
         imageStateProcessor = BehaviorProcessor.create();
+    }
+
+    public void setAppStateService(AppStateService appStateService) {
+        this.appStateService = appStateService;
         appStateService.getCurrentImages().put(this.url, this);
         appStateService.getLiveImageUpdates().onNext(this);
-        init();
     }
 
     public void setStatus(Status status) {
@@ -60,12 +68,11 @@ public class Image {
     }
 
     public void init() {
-        if (imageStateProcessor != null) {
-            imageStateProcessor.onComplete();
-        }
-        if (subscription != null) {
-            subscription.dispose();
+        if (appStateService == null) {
+            logger.warn("Attempting to init the image whilst the App State is null, unexpected behaviour could occur from this");
+            return;
         }
+        cleanup();
         imageStateProcessor = BehaviorProcessor.create();
         subscription = imageStateProcessor
                 .onBackpressureLatest()
@@ -73,10 +80,19 @@ public class Image {
                 .subscribe();
 
         current.set(0);
-        status = Status.PENDING;
+        status = Status.STOPPED;
         imageStateProcessor.onNext(this);
     }
 
+    public void cleanup() {
+        if (imageStateProcessor != null) {
+            imageStateProcessor.onComplete();
+        }
+        if (subscription != null) {
+            subscription.dispose();
+        }
+    }
+
     public void setCurrent(int current) {
         this.current.set(current);
         update();

vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java

@@ -2,21 +2,22 @@ package tn.mnlr.vripper.entities;
 
 import lombok.Getter;
 import lombok.NoArgsConstructor;
-import lombok.Setter;
 import lombok.ToString;
+import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 @Getter
 @ToString
 @NoArgsConstructor
 public class Post {
 
-    @Setter
     private AppStateService appStateService;
 
     private Status status;
@@ -25,45 +26,64 @@ public class Post {
 
     private String postId;
 
+    private String threadTitle;
+
+    private String threadId;
+
     private String title;
 
     private String url;
 
     private List<Image> images;
 
-    private Map<String, String> metadata;
+    private Map<String, Object> metadata;
 
     private AtomicInteger done = new AtomicInteger(0);
 
     private int total;
 
+    private Set<String> hosts;
+
     private boolean removed = false;
 
-    public Post(String title, String url, List<Image> images, Map<String, String> metadata, String postId, AppStateService appStateService) {
+    private String forum;
+
+    public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) {
         this.title = title;
         this.url = url;
         this.images = images;
         this.metadata = metadata;
         this.postId = postId;
-        this.appStateService = appStateService;
+        this.forum = forum;
+        this.threadId = threadId;
+        this.threadTitle = threadTitle;
+        this.hosts = this.images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet());
         total = images.size();
-        status = Status.PENDING;
-        appStateService.getCurrentPosts().put(postId, this);
-        appStateService.getLivePostsState().onNext(this);
+        status = Status.STOPPED;
+    }
+
+    public void setAppStateService(AppStateService appStateService) {
+        this.appStateService = appStateService;
+        this.appStateService.getCurrentPosts().put(postId, this);
+        this.appStateService.getLivePostsState().onNext(this);
     }
 
     public void setRemoved(boolean removed) {
         this.removed = removed;
-        appStateService.getLivePostsState().onNext(this);
+        updateNotification();
     }
 
     public void increase() {
         done.incrementAndGet();
-        appStateService.getLivePostsState().onNext(this);
+        updateNotification();
     }
 
     public void setStatus(Status status) {
         this.status = status;
+        updateNotification();
+    }
+
+    private void updateNotification() {
         if (appStateService != null) {
             appStateService.getLivePostsState().onNext(this);
         }

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/persistance/ImagePersistanceMixin.java

@@ -5,10 +5,14 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.BehaviorProcessor;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class ImagePersistanceMixin {
 
     @JsonSerialize(converter = HostToString.class)

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/persistance/PostPersistanceMixin.java

@@ -1,8 +1,12 @@
 package tn.mnlr.vripper.entities.mixin.persistance;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class PostPersistanceMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/ui/ImageUIMixin.java

@@ -3,10 +3,14 @@ package tn.mnlr.vripper.entities.mixin.ui;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.processors.BehaviorProcessor;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
+@Getter
+@Setter
 public abstract class ImageUIMixin {
 
     @JsonIgnore

vripper-server/src/main/java/tn/mnlr/vripper/entities/mixin/ui/PostUIMixin.java

@@ -1,20 +1,20 @@
 package tn.mnlr.vripper.entities.mixin.ui;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.List;
-import java.util.Map;
 
+@Getter
+@Setter
 public abstract class PostUIMixin {
 
     @JsonIgnore
     private List<Image> images;
 
-    @JsonIgnore
-    private Map<String, String> metadata;
-
     @JsonIgnore
     private AppStateService appStateService;
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/DownloadException.java

@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
         super(message);
     }
 
-    public DownloadException(Exception e) {
+    public DownloadException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class HostException extends Exception {
-    public HostException(Exception e) {
+    public HostException(Throwable e) {
         super(e);
     }
 
@@ -9,7 +9,7 @@ public class HostException extends Exception {
         super(message);
     }
 
-    public HostException(String message, Exception e) {
+    public HostException(String message, Throwable e) {
         super(message,e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HtmlProcessorException.java

@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
 
 public class HtmlProcessorException extends Exception {
 
-    public HtmlProcessorException(Exception e) {
+    public HtmlProcessorException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/PostParseException.java

@@ -1,7 +1,16 @@
 package tn.mnlr.vripper.exception;
 
 public class PostParseException extends Exception {
-    public PostParseException(Exception e) {
+
+    public PostParseException(String message) {
+        super(message);
+    }
+
+    public PostParseException(String message, Throwable e) {
+        super(message, e);
+    }
+
+    public PostParseException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/VripperException.java

@@ -5,7 +5,7 @@ public class VripperException extends Exception {
         super(message);
     }
 
-    public VripperException(Exception e) {
+    public VripperException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/XpathException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class XpathException extends Exception {
-    public XpathException(Exception e) {
+    public XpathException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java

@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -28,31 +29,36 @@ public class AcidimgHost extends Host {
 
     private static final String host = "acidimg.cc";
     private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
-    public static final String IMG_XPATH = "//img[@class='centred']";
+    private static final String IMG_XPATH = "//img[@class='centred']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node contDiv;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
-            logger.info(String.format("Click button found for %s", url));
+            logger.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpPost httpPost = cm.buildHttpPost(url);
             httpPost.addHeader("Referer", url);
@@ -64,9 +70,9 @@ public class AcidimgHost extends Host {
                 throw new HostException(e);
             }
 
-            logger.info(String.format("Requesting %s", httpPost));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
-                logger.info(String.format("Cleaning response for %s", httpPost));
+            logger.debug(String.format("Requesting %s", httpPost));
+            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
+                logger.debug(String.format("Cleaning response for %s", httpPost));
                 doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
                 EntityUtils.consumeQuietly(response.getEntity());
             } catch (Exception e) {
@@ -76,7 +82,7 @@ public class AcidimgHost extends Host {
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
@@ -87,7 +93,7 @@ public class AcidimgHost extends Host {
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -5,27 +5,30 @@ import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Document;
-import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.HostException;
 import tn.mnlr.vripper.exception.HtmlProcessorException;
 import tn.mnlr.vripper.q.ImageFileData;
-import tn.mnlr.vripper.services.ConnectionManager;
-import tn.mnlr.vripper.services.HtmlProcessorService;
-import tn.mnlr.vripper.services.XpathService;
+import tn.mnlr.vripper.services.*;
 
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Random;
+import java.nio.file.Files;
+import java.util.Iterator;
 
 @Service
 abstract public class Host {
@@ -43,118 +46,140 @@ abstract public class Host {
     @Autowired
     private AppSettingsService appSettingsService;
 
+    @Autowired
+    private AppStateService appStateService;
+
     @Autowired
     private ConnectionManager cm;
 
-    abstract protected String getHost();
+    @Autowired
+    private DownloadSpeedService downloadSpeedService;
+
+    @Autowired
+    private PathService pathService;
+
+    abstract public String getHost();
+
+    abstract public String getLookup();
 
     public boolean isSupported(String url) {
-        return url.contains(getHost());
+        return url.contains(getLookup());
     }
 
     public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
 
+        HttpClientContext context = HttpClientContext.create();
+        context.setCookieStore(new BasicCookieStore());
         try {
 
+            appStateService.postDownloadingUpdate(image.getPostId());
+
             imageFileData.setPageUrl(image.getUrl());
 
-            /**
+            /*
              * HOST SPECIFIC
              */
-            logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
-            setNameAndUrl(image.getUrl(), imageFileData);
-            logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
-            logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
+            logger.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
+            setNameAndUrl(image.getUrl(), imageFileData, context);
+            logger.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
+            logger.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
 
-            logger.info(String.format("Building image request for %s", image.getUrl()));
+            logger.debug(String.format("Building image request for %s", image.getUrl()));
             setImageRequest(imageFileData);
-            /**
+            /*
              * END HOST SPECIFIC
              */
 
-            if(!imageFileData.getImageName().toLowerCase().endsWith(".jpg")) {
-                imageFileData.setImageName(imageFileData.getImageName() + ".jpg");
-            }
-
-            File destinationFolder = new File(appSettingsService.getDownloadPath(), sanitize(image.getPostName()));
-            logger.info(String.format("Saving to %s", destinationFolder.getPath()));
+            String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
+            logger.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
+            imageFileData.setImageName(formatImageFileName);
+            File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
+            logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
             if (!destinationFolder.exists()) {
-                logger.info(String.format("Creating %s", destinationFolder.getPath()));
-                destinationFolder.mkdirs();
-            } else {
-                logger.warn(String.format("%s already exists, the file will be overridden", destinationFolder.getPath()));
+                logger.debug(String.format("Creating %s", destinationFolder.getPath()));
+                if (destinationFolder.mkdirs()) {
+                    logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
+                }
             }
 
             HttpClient client = cm.getClient().build();
 
-            logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
+            logger.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
+            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest(), context)) {
 
-                if(response.getStatusLine().getStatusCode() / 100 != 2) {
+                if (response.getStatusLine().getStatusCode() / 100 != 2) {
                     EntityUtils.consumeQuietly(response.getEntity());
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
-                try (
-                        InputStream downloadStream = response.getEntity().getContent();
-                        FileOutputStream fos = new FileOutputStream(destinationFolder.getPath() + File.separator + imageFileData.getImageName())
-                ) {
-
+                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
+                try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
                     image.setTotal(response.getEntity().getContentLength());
-                    logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
-                    logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
+                    logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
+                    logger.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
 
                     byte[] buffer = new byte[READ_BUFFER_SIZE];
                     int read;
                     while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
                         fos.write(buffer, 0, read);
                         image.increase(read);
+                        downloadSpeedService.increase(read);
                     }
                     EntityUtils.consumeQuietly(response.getEntity());
                 }
+                checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
             }
         } catch (Exception e) {
-            if(Thread.interrupted()) {
+            if (Thread.interrupted()) {
                 throw new InterruptedException("Download was interrupted");
             }
             throw new DownloadException(e);
         }
     }
 
-    /**
-     * Just for testing, you may ignore
-     * @throws Exception
-     */
-    private void randomFail() throws Exception {
-        Random random = new Random();
-        int i = random.nextInt(100);
-        if (i < 50) {
-            throw new Exception("Purpose error");
+    private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
+        String formatName;
+        try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
+            Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
+            if (!it.hasNext()) {
+                throw new HostException("Image file is not recognized!");
+            }
+            ImageReader reader = it.next();
+            formatName = reader.getFormatName();
+            if (formatName.toUpperCase().equals("JPEG")) {
+                formatName = "jpg";
+            }
+        } catch (Exception e) {
+            throw new HostException("Failed to guess image format", e);
+        }
+        try {
+            File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
+            if (outImage.exists() && outImage.delete()) {
+                logger.debug(String.format("%s is deleted", outImage.toString()));
+            }
+            Files.move(outputFile.toPath(), outImage.toPath());
+        } catch (Exception e) {
+            throw new HostException("Failed to rename the image", e);
         }
     }
 
-
-
-    protected final String sanitize(final String folderName) {
-        String sanitizedFolderName = folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
-        logger.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
-        return sanitizedFolderName;
-    }
-
-    protected final String getDefaultImageName(final String imgUrl) {
-        String imageTitle  = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
+    final String getDefaultImageName(final String imgUrl) {
+        String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
         logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
         return imgUrl;
     }
 
-    protected final Response getResponse(final String url) throws HostException {
+    final Response getResponse(final String url, final HttpClientContext context) throws HostException {
         String basePage;
 
         HttpClient client = cm.getClient().build();
         HttpGet httpGet = cm.buildHttpGet(url);
         Header[] headers;
-        logger.info(String.format("Requesting %s", url));
-        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
+        logger.debug(String.format("Requesting %s", url));
+        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
+            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
+            }
             headers = response.getAllHeaders();
             basePage = EntityUtils.toString(response.getEntity());
             logger.debug(String.format("%s response: %n%s", url, basePage));
@@ -164,20 +189,20 @@ abstract public class Host {
         }
 
         try {
-            logger.info(String.format("Cleaning %s response", url));
+            logger.debug(String.format("Cleaning %s response", url));
             return new Response(htmlProcessorService.clean(basePage), headers);
         } catch (HtmlProcessorException e) {
             throw new HostException(e);
         }
     }
 
-    protected void setImageRequest(final ImageFileData imageFileData) {
+    private void setImageRequest(final ImageFileData imageFileData) {
         HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
         httpGet.addHeader("Referer", imageFileData.getPageUrl());
         imageFileData.setImageRequest(httpGet);
     }
 
-    protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
+    protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
 
     @Getter
     public static class Response {
@@ -185,7 +210,13 @@ abstract public class Host {
             this.document = document;
             this.headers = headers;
         }
+
         private Document document;
         private Header[] headers;
     }
+
+    @Override
+    public String toString() {
+        return getHost();
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageBamHost.java

@@ -1,9 +1,9 @@
 package tn.mnlr.vripper.host;
 
-import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -18,8 +18,6 @@ import tn.mnlr.vripper.q.ImageFileData;
 import tn.mnlr.vripper.services.ConnectionManager;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.stream.Collectors;
 
 @Service
 public class ImageBamHost extends Host {
@@ -27,46 +25,43 @@ public class ImageBamHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
 
     private static final String host = "imagebam.com";
-    public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
-    public static final String IMG_XPATH = "//img[@class='image']";
+    private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
+    private static final String IMG_XPATH = "//img[@class='image']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Response response = getResponse(url);
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Response response = getResponse(url, context);
         Document doc = response.getDocument();
-        Header[] headers = response.getHeaders();
-        String cookies = Arrays.asList(headers)
-                .stream()
-                .filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
-                .map(e -> e.getValue().split(";")[0].trim())
-                .collect(Collectors.joining("; "));
 
         Node contDiv;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
-            logger.info(String.format("Getting cookies to use for %s", url));
-            logger.info(String.format("Click button found for %s", url));
+            logger.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpGet httpGet = cm.buildHttpGet(url);
             httpGet.addHeader("Referer", url);
-            httpGet.addHeader("Cookie", cookies);
-            logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
+            logger.debug(String.format("Requesting %s", httpGet));
+            try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
                 String s = EntityUtils.toString(res.getEntity());
                 logger.debug(String.format("%s response is:%n%s", httpGet, s));
                 logger.debug(String.format("Cleaning response for %s", httpGet));
@@ -79,14 +74,14 @@ public class ImageBamHost extends Host {
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageTwistHost.java

@@ -0,0 +1,60 @@
+package tn.mnlr.vripper.host;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import tn.mnlr.vripper.exception.HostException;
+import tn.mnlr.vripper.exception.XpathException;
+import tn.mnlr.vripper.q.ImageFileData;
+import tn.mnlr.vripper.services.ConnectionManager;
+
+import java.util.Optional;
+
+@Service
+public class ImageTwistHost extends Host {
+
+    private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
+    private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
+    private static final String host = "imagetwist.com";
+    @Autowired
+    private ConnectionManager cm;
+
+    @Override
+    public String getHost() {
+        return host;
+    }
+
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
+
+        Node imgNode;
+        try {
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            imgNode = xpathService.getAsNode(doc, IMG_XPATH);
+        } catch (XpathException e) {
+            throw new HostException(e);
+        }
+
+        try {
+            logger.debug(String.format("Resolving name and image url for %s", url));
+            String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
+            String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
+
+            imageFileData.setImageUrl(imgUrl);
+            imageFileData.setImageName(imgTitle);
+        } catch (Exception e) {
+            throw new HostException("Unexpected error occurred", e);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,26 +18,32 @@ public class ImageZillaHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
 
     private static final String host = "imagezilla.net";
-    public static final String IMG_XPATH = "//img[@id='photo']";
+    private static final String lookup = "imagezilla.net/show";
+    private static final String IMG_XPATH = "//img[@id='photo']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return lookup;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         String title;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
-            logger.info(String.format("Resolving name for %s", url));
+            logger.debug(String.format("Resolving name for %s", url));
             if(titleNode != null) {
                 title = titleNode.getTextContent().trim();
             } else {

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,36 +18,41 @@ public class ImgboxHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
 
     private static final String host = "imgbox.com";
-    public static final String IMG_XPATH = "//img[@id='img']";
+    private static final String IMG_XPATH = "//img[@id='img']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 
             imageFileData.setImageUrl(imgUrl);
-            imageFileData.setImageName(imgTitle.substring(8));
+            imageFileData.setImageName(imgTitle);
         } catch (Exception e) {
             throw new HostException("Unexpected error occurred", e);
         }

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -29,62 +30,74 @@ public class ImxHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
 
     private static final String host = "imx.to";
-    public static final String CONTINUE_BUTTON_XPATH = "//div[@id='continuetoimage']";
-    public static final String IMG_XPATH = "//img[@class='centred']";
+    private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
+    private static final String IMG_XPATH = "//img[@class='centred']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url, context);
+        Document doc = resp.getDocument();
 
         Node contDiv;
+        String value = null;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
+            Node node = contDiv.getAttributes().getNamedItem("value");
+            if (node != null) {
+                value = node.getTextContent();
+            }
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
-        if (contDiv != null) {
-            logger.info(String.format("Click button found for %s", url));
-            HttpClient client = cm.getClient().build();
-            HttpPost httpPost = cm.buildHttpPost(url);
-            httpPost.addHeader("Referer", url);
-            List<NameValuePair> params = new ArrayList<>();
-            params.add(new BasicNameValuePair("imgContinue", "Continue to image ... "));
-            try {
-                httpPost.setEntity(new UrlEncodedFormEntity(params));
-            } catch (Exception e) {
-                throw new HostException(e);
-            }
-            logger.info(String.format("Requesting %s", httpPost));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
-                logger.debug(String.format("Cleaning response for %s", httpPost));
-                doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (IOException | HtmlProcessorException e) {
-                throw new HostException(e);
-            }
+        if (value == null) {
+            throw new HostException("Failed to obtain value attribute from continue input");
+        }
+        logger.debug(String.format("Click button found for %s", url));
+        HttpClient client = cm.getClient().build();
+        HttpPost httpPost = cm.buildHttpPost(url);
+        List<NameValuePair> params = new ArrayList<>();
+        params.add(new BasicNameValuePair("imgContinue", value));
+        try {
+            httpPost.setEntity(new UrlEncodedFormEntity(params));
+        } catch (Exception e) {
+            throw new HostException(e);
+        }
+        logger.debug(String.format("Requesting %s", httpPost));
+        try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
+            logger.debug(String.format("Cleaning response for %s", httpPost));
+            doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
+            EntityUtils.consumeQuietly(response.getEntity());
+        } catch (IOException | HtmlProcessorException e) {
+            throw new HostException(e);
         }
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,31 +18,37 @@ public class PixhostHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
 
     private static final String host = "pixhost.to";
-    public static final String IMG_XPATH = "//img[@id='image']";
+    private static final String lookup = "pixhost.to/show";
+    private static final String IMG_XPATH = "//img[@id='image']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return lookup;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         Node imgNode;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
             imgNode = xpathService.getAsNode(doc, IMG_XPATH);
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         try {
-            logger.info(String.format("Resolving name and image url for %s", url));
+            logger.debug(String.format("Resolving name and image url for %s", url));
             String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
             String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
 

vripper-server/src/main/java/tn/mnlr/vripper/host/TurboImageHost.java

@@ -1,5 +1,6 @@
 package tn.mnlr.vripper.host;
 
+import org.apache.http.client.protocol.HttpClientContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,27 +18,32 @@ public class TurboImageHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
 
     private static final String host = "turboimagehost.com";
-    public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
-    public static final String IMG_XPATH = "//img[@id='uImage']";
+    private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
+    private static final String IMG_XPATH = "//img[@id='uImage']";
 
     @Autowired
     private ConnectionManager cm;
 
     @Override
-    protected String getHost() {
+    public String getHost() {
         return host;
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
+
+        Document doc = getResponse(url, context).getDocument();
 
         String title;
         try {
-            logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
+            logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
             Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
-            logger.info(String.format("Resolving name for %s", url));
+            logger.debug(String.format("Resolving name for %s", url));
             if(titleNode != null) {
                 title = titleNode.getTextContent().trim();
             } else {

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java

@@ -18,14 +18,14 @@ public class DownloadJob implements Callable<Image> {
     @Getter
     private final ImageFileData imageFileData = new ImageFileData();
 
-    public DownloadJob(Image image) {
+    DownloadJob(Image image) {
         this.image = image;
     }
 
     @Override
     public Image call() throws Exception {
 
-        logger.info(String.format("Starting downloading %s", image.getUrl()));
+        logger.debug(String.format("Starting downloading %s", image.getUrl()));
         image.setStatus(Image.Status.DOWNLOADING);
         image.setCurrent(0);
         image.getHost().download(image, imageFileData);

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java

@@ -9,6 +9,7 @@ import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.services.AppStateService;
 
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
@@ -18,6 +19,8 @@ import java.util.stream.Collectors;
 @Service
 public class DownloadQ {
 
+    private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
+
     private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
 
     @Autowired
@@ -31,79 +34,34 @@ public class DownloadQ {
     @Getter
     private boolean notPauseQ = true;
 
-    public synchronized void put(Image image) throws InterruptedException {
-        logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
-        DownloadJob downloadJob = new DownloadJob(image);
-        downloadQ.put(downloadJob);
-        appStateService.newDownloadJob(downloadJob);
+    public void put(Image image) throws InterruptedException {
+        synchronized (appStateService) {
+            logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
+            image.init();
+            DownloadJob downloadJob = new DownloadJob(image);
+            downloadQ.put(downloadJob);
+            appStateService.newDownloadJob(downloadJob);
+        }
     }
 
-    public DownloadJob take() throws InterruptedException {
+    DownloadJob take() throws InterruptedException {
         DownloadJob downloadJob = downloadQ.take();
-        logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
+        logger.debug(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
         return downloadJob;
     }
 
-    public void enqueue(Post post) throws InterruptedException {
-
+    public synchronized void enqueue(Post post) throws InterruptedException {
         for (Image image : post.getImages()) {
             put(image);
         }
     }
 
     public void restart(String postId) throws InterruptedException {
-        if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
-            logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
-            return;
-        }
-        List<Image> images = appStateService.getPost(postId)
-                .getImages()
-                .stream()
-                .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
-                .collect(Collectors.toList());
-        if (images.isEmpty()) {
-            return;
-        }
-        appStateService.getPost(postId).setStatus(Post.Status.PENDING);
-        logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
-        for (Image image : images) {
-            image.init();
-            put(image);
-        }
-    }
-
-    public void removeScheduled(Image image) {
-        image.setStatus(Image.Status.STOPPED);
-        logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
-
-        Iterator<DownloadJob> iterator = downloadQ.iterator();
-        boolean removed = false;
-        while(iterator.hasNext()) {
-            DownloadJob next = iterator.next();
-            if(next.getImage().getPostId().equals(image.getPostId())) {
-                iterator.remove();
-                appStateService.doneDownloadJob(image);
-                logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
-                removed = true;
-                break;
+        synchronized (appStateService) {
+            if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
+                logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
+                return;
             }
-        }
-
-        if(!removed) {
-            logger.warn(String.format("Job for %s does not exist", image.getUrl()));
-        }
-    }
-
-    public void removeRunning(String postId) {
-        logger.info(String.format("Interrupting running jobs for post id %s", postId));
-        executionService.stop(postId);
-    }
-
-
-    public synchronized void stop(String postId) {
-        try {
-            notPauseQ = false;
-            appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
             List<Image> images = appStateService.getPost(postId)
                     .getImages()
                     .stream()
@@ -112,11 +70,87 @@ public class DownloadQ {
             if (images.isEmpty()) {
                 return;
             }
-            logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
-            images.forEach(image -> removeScheduled(image));
-            removeRunning(postId);
+            appStateService.getPost(postId).setStatus(Post.Status.PENDING);
+            logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
+            for (Image image : images) {
+                put(image);
+            }
+        }
+    }
+
+    private void removeScheduled(Image image) {
+        synchronized (appStateService) {
+            image.setStatus(Image.Status.STOPPED);
+            logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
+
+            Iterator<DownloadJob> iterator = downloadQ.iterator();
+            boolean removed = false;
+            while (iterator.hasNext()) {
+                DownloadJob next = iterator.next();
+                if (next.getImage().getPostId().equals(image.getPostId())) {
+                    iterator.remove();
+                    appStateService.doneDownloadJob(image);
+                    logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
+                    removed = true;
+                    break;
+                }
+            }
+
+            if (!removed) {
+                logger.debug(String.format("Job for %s does not exist", image.getUrl()));
+            }
+
+            image.cleanup();
+        }
+    }
+
+    private void removeRunning(String postId) {
+        logger.debug(String.format("Interrupting running jobs for post id %s", postId));
+        executionService.stop(postId);
+    }
+
+
+    public void stop(String postId) {
+        try {
+            synchronized (appStateService) {
+                if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
+                    return;
+                }
+                notPauseQ = false;
+                appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
+                List<Image> images = appStateService.getPost(postId)
+                        .getImages()
+                        .stream()
+                        .filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
+                        .collect(Collectors.toList());
+                if (images.isEmpty()) {
+                    return;
+                }
+                logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
+                images.forEach(this::removeScheduled);
+                removeRunning(postId);
+            }
         } finally {
             notPauseQ = true;
         }
     }
+
+    public int size() {
+        return downloadQ.size();
+    }
+
+    public void stopAll() {
+        synchronized (appStateService) {
+            appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
+        }
+    }
+
+    public void restartAll() throws InterruptedException {
+        synchronized (appStateService) {
+            for (Post post : appStateService.getCurrentPosts().values()) {
+                String postId = post.getPostId();
+                restart(postId);
+            }
+        }
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -7,13 +7,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
-import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
-import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -36,7 +36,7 @@ public class ExecutionService {
     @Autowired
     private AppStateService appStateService;
 
-    private AtomicInteger threadCount = new AtomicInteger();
+    private final AtomicInteger threadCount = new AtomicInteger();
 
     private ExecutorService executor = Executors.newFixedThreadPool(10);
 
@@ -53,8 +53,8 @@ public class ExecutionService {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withDelay(Duration.ofSeconds(20))
-                .withMaxRetries(2)
+                .withBackoff(10, 60, ChronoUnit.SECONDS)
+                .withMaxRetries(4)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 
@@ -68,7 +68,7 @@ public class ExecutionService {
         executionThread.interrupt();
         executor.shutdown();
         appStateService.getCurrentPosts().keySet().forEach(p -> {
-            logger.info(String.format("Stopping download jobs for %s", p));
+            logger.debug(String.format("Stopping download jobs for %s", p));
             this.stop(p);
         });
         executor.awaitTermination(10, TimeUnit.SECONDS);
@@ -80,17 +80,18 @@ public class ExecutionService {
                 .filter(e -> e.getImage().getPostId().equals(postId))
                 .peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
                 .collect(Collectors.toList());
-        logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
+        logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
 
         data.forEach(e -> {
             futures.get(e.getImage().getUrl()).cancel(true);
             if(e.getImageFileData().getImageRequest() != null) {
                 e.getImageFileData().getImageRequest().abort();
             }
+            e.getImage().cleanup();
         });
     }
 
-    boolean canRun() {
+    private boolean canRun() {
         boolean canRun = threadCount.get() < settings.getMaxThreads();
         if (canRun && downloadQ.isNotPauseQ()) {
             threadCount.incrementAndGet();
@@ -102,9 +103,12 @@ public class ExecutionService {
     public void start() {
         while (!Thread.interrupted()) {
             if (canRun()) {
-                DownloadJob take = null;
+                DownloadJob take;
                 try {
                     take = downloadQ.take();
+                    if (take == null) {
+                        continue;
+                    }
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
                     break;
@@ -117,7 +121,7 @@ public class ExecutionService {
                     Failsafe.with(retryPolicy)
                             .onFailure(e -> {
                                 if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
-                                    logger.info("Job successfully interrupted");
+                                    logger.debug("Job successfully interrupted");
                                     return;
                                 }
                                 logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
@@ -125,9 +129,9 @@ public class ExecutionService {
                             })
                             .onComplete(e -> {
                                 appStateService.doneDownloadJob(finalTake.getImage());
-                                logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
+                                logger.debug(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
                                 synchronized (threadCount) {
-                                    int i = threadCount.decrementAndGet();
+                                    threadCount.decrementAndGet();
                                     running.remove(finalTake);
                                     futures.remove(finalTake.getImage().getUrl());
                                     threadCount.notify();
@@ -135,7 +139,7 @@ public class ExecutionService {
                             })
                             .get(finalTake::call);
                 };
-                logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
+                logger.debug(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
                 futures.put(finalTake.getImage().getUrl(), executor.submit(task));
             } else {
                 synchronized (threadCount) {
@@ -149,4 +153,8 @@ public class ExecutionService {
             }
         }
     }
+
+    public int runningCount() {
+        return running.size();
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -2,6 +2,7 @@ package tn.mnlr.vripper.services;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
@@ -9,6 +10,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.exception.ValidationException;
 
+import javax.annotation.PreDestroy;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
@@ -33,6 +35,11 @@ public class AppSettingsService {
     private final String V_PASSWORD = "VPASSWORD";
     private final String V_THANKS = "VTHANKS";
     private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
+    private final String FORCE_ORDER = "FORCE_ORDER";
+    private final String SUBFOLDER = "SUBFOLDER";
+    private final String THREADSUBFOLDER = "THREADSUBFOLDER";
+    private final String CLEAR = "CLEAR";
+    private final String DARK_THEME = "DARK_THEME";
 
     private String downloadPath;
     private int maxThreads;
@@ -42,6 +49,11 @@ public class AppSettingsService {
     private String vPassword;
     private boolean vThanks;
     private boolean desktopClipboard;
+    private boolean subLocation;
+    private boolean threadSubLocation;
+    private boolean forceOrder;
+    private boolean clearCompleted;
+    private boolean darkTheme;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -61,8 +73,14 @@ public class AppSettingsService {
         vPassword = prefs.get(V_PASSWORD, "");
         vThanks = prefs.getBoolean(V_THANKS, false);
         desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
+        forceOrder = prefs.getBoolean(FORCE_ORDER, false);
+        subLocation = prefs.getBoolean(SUBFOLDER, false);
+        threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
+        clearCompleted = prefs.getBoolean(CLEAR, false);
+        darkTheme = prefs.getBoolean(DARK_THEME, false);
     }
 
+    @PreDestroy
     public void save() {
 
         prefs.put(DOWNLOAD_PATH, downloadPath);
@@ -73,6 +91,11 @@ public class AppSettingsService {
         prefs.put(V_PASSWORD, vPassword);
         prefs.putBoolean(V_THANKS, vThanks);
         prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
+        prefs.putBoolean(FORCE_ORDER, forceOrder);
+        prefs.putBoolean(SUBFOLDER, subLocation);
+        prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
+        prefs.putBoolean(CLEAR, clearCompleted);
+        prefs.putBoolean(DARK_THEME, darkTheme);
 
         try {
             prefs.sync();
@@ -95,8 +118,29 @@ public class AppSettingsService {
             throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
         }
 
-        if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 8) {
-            throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 8));
+        if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
+            throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
+        }
+    }
+
+    public Theme getTheme() {
+        return new Theme(this.darkTheme);
+    }
+
+    public void setTheme(Theme theme) {
+        this.darkTheme = theme.darkTheme;
+        save();
+    }
+
+    @Getter
+    @NoArgsConstructor
+    public static class Theme {
+
+        @JsonProperty("darkTheme")
+        private boolean darkTheme;
+
+        Theme(boolean darkTheme) {
+            this.darkTheme = darkTheme;
         }
     }
 
@@ -119,8 +163,16 @@ public class AppSettingsService {
         private boolean vThanks;
         @JsonProperty("desktopClipboard")
         private boolean desktopClipboard;
+        @JsonProperty("forceOrder")
+        private boolean forceOrder;
+        @JsonProperty("subLocation")
+        private boolean subLocation;
+        @JsonProperty("threadSubLocation")
+        private boolean threadSubLocation;
+        @JsonProperty("clearCompleted")
+        private boolean clearCompleted;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard) {
+        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
             this.autoStart = autoStart;
@@ -129,6 +181,10 @@ public class AppSettingsService {
             this.vPassword = vPassword;
             this.vThanks = vThanks;
             this.desktopClipboard = desktopClipboard;
+            this.forceOrder = forceOrder;
+            this.subLocation = subLocation;
+            this.threadSubLocation = threadSubLocation;
+            this.clearCompleted = clearCompleted;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -2,14 +2,15 @@ package tn.mnlr.vripper.services;
 
 import io.reactivex.processors.PublishProcessor;
 import lombok.Getter;
+import lombok.Setter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
-import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.q.DownloadJob;
 
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -20,6 +21,8 @@ import java.util.stream.Collectors;
 @Getter
 public class AppStateService {
 
+    private static final Logger logger = LoggerFactory.getLogger(AppStateService.class);
+
     private Map<String, Image> currentImages = new ConcurrentHashMap<>();
 
     private Map<String, Post> currentPosts = new ConcurrentHashMap<>();
@@ -33,6 +36,9 @@ public class AppStateService {
     @Autowired
     private PersistenceService persistenceService;
 
+    @Autowired
+    private AppSettingsService appSettingsService;
+
     public void onImageUpdate(Image imageState) {
 
         persistenceService.getProcessor().onNext(currentPosts);
@@ -45,14 +51,6 @@ public class AppStateService {
         }
     }
 
-    public Image createImage(String pageUrl, String postId, String postName, Host host) {
-        return new Image(pageUrl, postId, postName, host, this);
-    }
-
-    public Post createPost(String title, String url, List<Image> images, Map<String, String> metadata, String postId) {
-        return new Post(title, url, images, metadata, postId, this);
-    }
-
     public Post getPost(String postId) {
         return currentPosts.get(postId);
     }
@@ -60,9 +58,12 @@ public class AppStateService {
     public synchronized void newDownloadJob(DownloadJob downloadJob) {
         String postId = downloadJob.getImage().getPostId();
         checkKeyRunningPosts(postId);
-        int i = runningPosts.get(postId).incrementAndGet();
-        if (i > 0) {
-            Post post = currentPosts.get(postId);
+        runningPosts.get(postId).incrementAndGet();
+    }
+
+    public synchronized void postDownloadingUpdate(String postId) {
+        Post post = currentPosts.get(postId);
+        if (!post.getStatus().equals(Post.Status.DOWNLOADING)) {
             post.setStatus(Post.Status.DOWNLOADING);
             livePostsState.onNext(post);
         }
@@ -76,12 +77,16 @@ public class AppStateService {
             post.setStatus(Post.Status.PARTIAL);
         }
         if (i == 0) {
-            if (post.getImages().stream().map(Image::getStatus).filter(e -> e.equals(Image.Status.ERROR)).count() > 0) {
+            if (post.getImages().stream().map(Image::getStatus).anyMatch(e -> e.equals(Image.Status.ERROR))) {
                 post.setStatus(Post.Status.ERROR);
             } else {
-                post.setStatus(Post.Status.COMPLETE);
+                if (!Post.Status.STOPPED.equals(post.getStatus())) {
+                    post.setStatus(Post.Status.COMPLETE);
+                    if (appSettingsService.isClearCompleted()) {
+                        remove(image.getPostId());
+                    }
+                }
             }
-            livePostsState.onNext(post);
         }
     }
 
@@ -95,13 +100,7 @@ public class AppStateService {
         currentPosts.get(postId).setRemoved(true);
         runningPosts.remove(postId);
         currentPosts.remove(postId);
-        Iterator<Map.Entry<String, Image>> iterator = currentImages.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, Image> entry = iterator.next();
-            if(entry.getValue().getPostId().equals(postId)) {
-                iterator.remove();
-            }
-        }
+        currentImages.entrySet().removeIf(entry -> entry.getValue().getPostId().equals(postId));
         persistenceService.getProcessor().onNext(currentPosts);
     }
 
@@ -125,4 +124,15 @@ public class AppStateService {
         toRemove.forEach(this::remove);
         return toRemove;
     }
+
+
+    @Getter
+    public static class CachedThread {
+
+        private Map<String, Post> posts = new ConcurrentHashMap<>();
+        private AtomicInteger parsed = new AtomicInteger(0);
+
+        @Setter
+        private int total = 0;
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -4,13 +4,19 @@ import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
+import java.net.URI;
+import java.util.concurrent.TimeUnit;
+
 @Service
+@EnableScheduling
 public class ConnectionManager {
 
     private ConnectionManager() {
@@ -29,14 +35,21 @@ public class ConnectionManager {
     private void buildConnectionPool() {
 
         pcm = new PoolingHttpClientConnectionManager();
-        pcm.setMaxTotal(200);
-        pcm.setDefaultMaxPerRoute(10);
+        pcm.setMaxTotal(50);
+        pcm.setDefaultMaxPerRoute(4);
+    }
+
+    @Scheduled(fixedDelay = 1000)
+    private void idleConnectionMonitoring() {
+        pcm.closeExpiredConnections();
+        pcm.closeIdleConnections(30, TimeUnit.SECONDS);
     }
 
     public HttpClientBuilder getClient() {
         return HttpClients.custom()
                 .setConnectionManager(pcm)
-                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
+                .setRedirectStrategy(new LaxRedirectStrategy())
+                .disableAutomaticRetries()
                 .setDefaultRequestConfig(rc);
     }
 
@@ -51,4 +64,10 @@ public class ConnectionManager {
         httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
         return httpPost;
     }
+
+    public HttpGet buildHttpGet(URI uri) {
+        HttpGet httpGet = new HttpGet(uri);
+        httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
+        return httpGet;
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeed.java

@@ -0,0 +1,26 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+
+@Getter
+public class DownloadSpeed {
+
+    private final String type = "downSpeed";
+    private String speed;
+
+    public DownloadSpeed(long bytes) {
+        speed = formatSI(bytes);
+    }
+
+    private String formatSI(long bytes) {
+        return humanReadableByteCount(bytes, true);
+    }
+
+    private String humanReadableByteCount(long bytes, boolean si) {
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeedService.java

@@ -0,0 +1,40 @@
+package tn.mnlr.vripper.services;
+
+import io.reactivex.processors.PublishProcessor;
+import lombok.Getter;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+@Service
+@EnableScheduling
+public class DownloadSpeedService {
+
+    private AtomicLong read = new AtomicLong(0);
+
+    private long currentValue;
+
+    @Getter
+    private PublishProcessor<Long> readBytesPerSecond = PublishProcessor.create();
+
+    private boolean allowWrite = false;
+
+    public void increase(long read) {
+        if (allowWrite) {
+            this.read.addAndGet(read);
+        }
+    }
+
+    @Scheduled(fixedDelay = 1000)
+    private void calc() {
+        allowWrite = false;
+        long newValue = read.getAndSet(0);
+        if (newValue != currentValue) {
+            currentValue = newValue;
+            readBytesPerSecond.onNext(currentValue);
+        }
+        allowWrite = true;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalState.java

@@ -0,0 +1,35 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public class GlobalState {
+
+    private final String type = "globalState";
+    private long running;
+    private long queued;
+    private long remaining;
+    private long error;
+
+    GlobalState(long running, long queued, long remaining, long error) {
+        this.running = running;
+        this.queued = queued;
+        this.remaining = remaining;
+        this.error = error;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        GlobalState that = (GlobalState) o;
+        return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(type, running, queued, remaining, error);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/GlobalStateService.java

@@ -0,0 +1,55 @@
+package tn.mnlr.vripper.services;
+
+import io.reactivex.processors.PublishProcessor;
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.q.DownloadQ;
+import tn.mnlr.vripper.q.ExecutionService;
+
+@Service
+@EnableScheduling
+public class GlobalStateService {
+
+    @Autowired
+    private DownloadQ downloadQ;
+
+    @Autowired
+    private ExecutionService executionService;
+
+    @Autowired
+    private AppStateService appStateService;
+
+    @Getter
+    private GlobalState currentState;
+
+    @Getter
+    private PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
+
+    public GlobalStateService() {
+    }
+
+    @Scheduled(fixedDelay = 3000)
+    private void interval() {
+        GlobalState newGlobalState = new GlobalState(
+                executionService.runningCount(),
+                downloadQ.size(),
+                appStateService.getCurrentImages()
+                        .values()
+                        .stream()
+                        .filter(e -> e.getTotal() == 0 || e.getTotal() != e.getCurrent().get())
+                        .count(),
+                appStateService.getCurrentImages()
+                        .values()
+                        .stream()
+                        .filter(e -> e.getStatus().equals(Image.Status.ERROR))
+                        .count());
+        if (!newGlobalState.equals(currentState)) {
+            currentState = newGlobalState;
+            liveGlobalState.onNext(currentState);
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/HtmlProcessorService.java

@@ -11,10 +11,6 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
 @Service
 public class HtmlProcessorService {
 
-    private HtmlProcessorService() {
-
-    }
-
     public Document clean(String htmlContent) throws HtmlProcessorException {
         try {
             TagNode clean = new HtmlCleaner().clean(htmlContent);

vripper-server/src/main/java/tn/mnlr/vripper/services/PathService.java

@@ -0,0 +1,53 @@
+package tn.mnlr.vripper.services;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.entities.Post;
+
+import java.io.File;
+
+@Service
+public class PathService {
+
+    private static final Logger logger = LoggerFactory.getLogger(PathService.class);
+
+    @Autowired
+    private AppStateService appStateService;
+
+    @Autowired
+    private AppSettingsService appSettingsService;
+
+    public final File getDownloadDestinationFolder(String postId) {
+        Post post = appStateService.getCurrentPosts().get(postId);
+        String postTitle = post.getTitle();
+        String threadTitle = post.getThreadTitle();
+        File sourceFolder = appSettingsService.isSubLocation() ? new File(appSettingsService.getDownloadPath(), sanitize(post.getForum())) : new File(appSettingsService.getDownloadPath());
+        sourceFolder = appSettingsService.isThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
+        return new File(sourceFolder, sanitize(postTitle + "_" + postId));
+    }
+
+    public final String sanitize(final String folderName) {
+        String sanitizedFolderName = folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
+        logger.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
+        return sanitizedFolderName;
+    }
+
+    /**
+     * Will sanitize the image name and remove extension
+     *
+     * @param imageName path string
+     * @return Sanitized local path string
+     */
+    public final String formatImageFileName(String imageName) {
+        int extensionIndex = imageName.lastIndexOf('.');
+        String fileName;
+        if (extensionIndex != -1) {
+            fileName = imageName.substring(0, extensionIndex);
+        } else {
+            fileName = imageName;
+        }
+        return sanitize(fileName);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java

@@ -8,24 +8,27 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
 import tn.mnlr.vripper.entities.mixin.persistance.PostPersistanceMixin;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @Service
 public class PersistenceService {
@@ -35,6 +38,9 @@ public class PersistenceService {
     @Autowired
     private AppStateService stateService;
 
+    @Autowired
+    private VripperApplication.AppCommandRunner appCommandRunner;
+
     private ObjectMapper om;
 
     private Disposable subscription;
@@ -43,19 +49,6 @@ public class PersistenceService {
     private PublishProcessor<Map<String, Post>> processor = PublishProcessor.create();
 
     private PersistenceService() {
-        File dataFile = new File(VripperApplication.dataPath);
-        if(!dataFile.exists()) {
-            try {
-                if(dataFile.createNewFile()) {
-                    logger.info("Data file successfully created");
-                } else {
-                    logger.info("Data file already exists");
-                }
-            } catch (IOException e) {
-                logger.error("Unable to create data file", e);
-                System.exit(-1);
-            }
-        }
         om = new ObjectMapper();
         om.addMixIn(Image.class, ImagePersistanceMixin.class);
         om.addMixIn(Post.class, PostPersistanceMixin.class);
@@ -68,6 +61,34 @@ public class PersistenceService {
                 .subscribe();
     }
 
+    @PostConstruct
+    public void init() {
+        File dataFile = new File(appCommandRunner.getDataPath());
+        if (!dataFile.exists()) {
+            try {
+                if (dataFile.getParentFile().mkdirs()) {
+                    logger.debug(String.format("%s is created", dataFile.getParentFile().toString()));
+                }
+                if (!dataFile.getParentFile().isDirectory() || !dataFile.getParentFile().canWrite()) {
+                    logger.error(String.format("Unable to write in %s", dataFile.getParent()));
+                    SpringContext.close();
+                }
+
+                if (dataFile.createNewFile()) {
+                    logger.debug("Data file successfully created");
+                    try (FileWriter fw = new FileWriter(dataFile)) {
+                        fw.write("{}");
+                    }
+                } else {
+                    logger.warn("Data file already exists");
+                }
+            } catch (IOException e) {
+                logger.error("Unable to create data file", e);
+                SpringContext.close();
+            }
+        }
+    }
+
     @PreDestroy
     public void preDestroy() {
         this.subscription.dispose();
@@ -76,9 +97,9 @@ public class PersistenceService {
         this.persist(stateService.getCurrentPosts());
     }
 
-    public void persist(Map<String, Post> currentPosts) {
+    private void persist(Map<String, Post> currentPosts) {
 
-        try (PrintWriter out = new PrintWriter(VripperApplication.dataPath, "UTF-8")) {
+        try (PrintWriter out = new PrintWriter(appCommandRunner.getDataPath(), StandardCharsets.UTF_8)) {
             out.print(om.writeValueAsString(currentPosts));
         } catch (IOException e) {
             logger.error("Failed to persist app state", e);
@@ -91,6 +112,14 @@ public class PersistenceService {
             return om.readValue(content, om.getTypeFactory().constructMapType(HashMap.class, String.class, Post.class));
         } catch (IOException e) {
             logger.error("Failed to read app state", e);
+            long timestamp = new Date().getTime();
+            logger.warn(String.format("trying to rename old data file from %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + "." + timestamp + ".old"));
+            try {
+                Files.move(new File(appCommandRunner.getDataPath()).toPath(), new File(appCommandRunner.getDataPath() + "." + timestamp + ".old").toPath());
+            } catch (IOException ex) {
+                logger.error(String.format("Failed to rename %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + ".old"));
+                SpringContext.close();
+            }
         }
 
         return new HashMap<>();
@@ -98,12 +127,12 @@ public class PersistenceService {
 
     public void restore() {
 
-        String jsonContent;
+        String jsonContent = null;
         try {
-            jsonContent = Files.readAllLines(Paths.get(VripperApplication.dataPath), Charset.forName("UTF-8")).stream().collect(Collectors.joining());
+            jsonContent = String.join("", Files.readAllLines(Paths.get(appCommandRunner.getDataPath()), StandardCharsets.UTF_8));
         } catch (Exception e) {
-            logger.warn("data file not found, previous state cannot be restored", e);
-            return;
+            logger.error("data file cannot be read, previous state cannot be restored", e);
+            SpringContext.close();
         }
 
         Map<String, Post> read = read(jsonContent);
@@ -111,7 +140,7 @@ public class PersistenceService {
         stateService.getCurrentPosts().clear();
         stateService.getCurrentPosts().putAll(read);
         stateService.getCurrentPosts().values().forEach(p -> {
-            if(Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL).contains(p.getStatus())) {
+            if (Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL, Post.Status.PENDING).contains(p.getStatus())) {
                 p.setStatus(Post.Status.STOPPED);
             }
         });
@@ -122,8 +151,6 @@ public class PersistenceService {
             stateService.getCurrentImages().put(e.getUrl(), e);
         });
 
-        read.values().forEach(e -> {
-            e.setAppStateService(stateService);
-        });
+        read.values().forEach(e -> e.setAppStateService(stateService));
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -1,23 +1,15 @@
 package tn.mnlr.vripper.services;
 
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
-import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.host.Host;
+import tn.mnlr.vripper.q.DownloadQ;
 
-import java.util.ArrayList;
+import javax.annotation.PostConstruct;
 import java.util.List;
 
 @Service
@@ -25,163 +17,66 @@ public class PostParser {
 
     private static final Logger logger = LoggerFactory.getLogger(PostParser.class);
 
-    private final static String VIPER_GIRLS_BASE_ADRESS = "https://vipergirls.to/";
-    private final static String POSTS_XPATH = "//li[contains(@id,'post_')][not(contains(@id,'post_thank'))]";
-    private final static String REAL_THREAD_XPATH = ".//a[@class='postcounter']";
-    private final static String THREAD_TITLE_XPATH = "//li[contains(@class, 'lastnavbit')]/span";
-    private final static String POST_TITLE_XPATH = ".//h2";
-    private final static String POST_LINKS_XPATH = ".//a";
+    static final String VR_API = "https://vipergirls.to/vr.php";
 
     @Autowired
     private ConnectionManager cm;
 
     @Autowired
-    private HtmlProcessorService htmlProcessorService;
+    private AppStateService appStateService;
 
     @Autowired
-    private XpathService xpathService;
+    private VipergirlsAuthService authService;
+
+    @Autowired
+    private AppSettingsService appSettingsService;
+
+    @Autowired
+    private DownloadQ downloadQ;
+
+    @Autowired
+    private VipergirlsAuthService vipergirlsAuthService;
 
     @Autowired
     private List<Host> supportedHosts;
 
-    @Autowired
-    private AppStateService appStateService;
+    @PostConstruct
+    private void init() {
 
-    public List<Post> parse(String threadUrl) throws PostParseException {
+    }
 
-        logger.info(String.format("Parsing thread %s", threadUrl));
-        List<Post> posts = new ArrayList<>();
-        String postResponse;
+    public void addPost(String postId, String threadId) throws PostParseException {
 
-        HttpClient connection = cm.getClient().build();
-        HttpGet httpGet = cm.buildHttpGet(threadUrl);
-
-        logger.info(String.format("Requesting %s", httpGet));
-        try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-            if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-            }
-            postResponse = EntityUtils.toString(response.getEntity());
-            logger.debug(String.format("Content received for %s:%n%s", httpGet, postResponse));
-            EntityUtils.consumeQuietly(response.getEntity());
-        } catch (Exception e) {
-            throw new PostParseException(e);
+        if (appStateService.getCurrentPosts().containsKey(postId)) {
+            logger.warn(String.format("skipping %s, already loaded", postId));
+            return;
         }
 
-        Document document;
-        try {
-            logger.debug(String.format("Cleaning HTML response for XML parsing: %s", postResponse));
-            document = htmlProcessorService.clean(postResponse);
-        } catch (Exception e) {
-            throw new PostParseException(e);
-        }
+        VRPostParser vrPostParser = new VRPostParser(threadId, postId, cm, vipergirlsAuthService, supportedHosts);
+        Post post = vrPostParser.parse();
 
-        String threadTitle = null;
-        try {
-            logger.info(String.format("Looking for thread title using xpath: %s", THREAD_TITLE_XPATH));
-            Node threadTitleNode = xpathService.getAsNode(document, THREAD_TITLE_XPATH);
-            if (threadTitleNode != null) {
-                threadTitle = threadTitleNode.getTextContent().trim();
-                logger.info(String.format("Thread title found %s", threadTitle));
-            }
-            if (threadTitle == null) {
-                logger.warn("Cannot find thread's title");
-            }
-        } catch (Exception e) {
-            throw new PostParseException(e);
-        }
-
-        NodeList postsNodeList;
-        try {
-            logger.info(String.format("Looking for posts using xpath: %s", POSTS_XPATH));
-            postsNodeList = xpathService.getAsNodeList(document, POSTS_XPATH);
-        } catch (Exception e) {
-            throw new PostParseException(e);
-        }
-
-        logger.info(String.format("Found %d posts in thread %s", postsNodeList.getLength(), threadUrl));
-
-        for (int i = 0; i < postsNodeList.getLength(); i++) {
-            String realUrl;
-            logger.info(String.format("Parsing post #%d", i + 1));
+        post.setAppStateService(appStateService);
+        post.getImages().forEach(e -> e.setAppStateService(appStateService));
+        authService.leaveThanks(post.getUrl(), post.getPostId());
+        if (appSettingsService.isAutoStart()) {
+            logger.debug("Auto start downloads option is enabled");
+            logger.debug(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
+            post.setStatus(Post.Status.PENDING);
             try {
-                logger.info(String.format("Finding posts's link"));
-                realUrl = VIPER_GIRLS_BASE_ADRESS.concat(xpathService
-                        .getAsNode(postsNodeList.item(i), REAL_THREAD_XPATH)
-                        .getAttributes()
-                        .getNamedItem("href")
-                        .getTextContent()
-                        .trim());
-                logger.info(String.format("Post's link: %s", realUrl));
-            } catch (Exception e) {
-                throw new PostParseException(e);
-            }
-
-            logger.info("Finding posts's id");
-            String postId = realUrl.substring(realUrl.indexOf("#")).replace("#post", "");
-            logger.info(String.format("Post's id: %s", postId));
-
-            if (appStateService.getCurrentPosts().containsKey(postId)) {
-                logger.warn(String.format("Post with id %s is already loaded, skipping", postId));
-                continue;
-            }
-
-            String postTitle;
-            try {
-                logger.info(String.format("Finding post's title"));
-                Node titleNode = xpathService.getAsNode(postsNodeList.item(i), POST_TITLE_XPATH);
-                if (titleNode != null) {
-                    postTitle = titleNode.getTextContent().trim();
-                    logger.info(String.format("Found post's title: %s", postTitle));
-                } else {
-                    logger.info("Cannot find post's title");
-                    if (threadTitle != null) {
-                        logger.info("Falling back to thread title to generate a post name");
-                        postTitle = threadTitle + "#" + postId;
-                    } else {
-                        logger.info("Falling back to post id to generate a post name");
-                        postTitle = "#" + postId;
-                    }
-                }
-            } catch (Exception e) {
-                throw new PostParseException(e);
-            }
-
-            ArrayList<Image> imagesList = new ArrayList<>();
-            try {
-                logger.info(String.format("Finding all links for post with id %s using xpath %s", postId, POST_LINKS_XPATH));
-                NodeList imagesNodeList = xpathService.getAsNodeList(postsNodeList.item(i), POST_LINKS_XPATH);
-                for (int j = 0; j < imagesNodeList.getLength(); j++) {
-
-                    Node imageHref = imagesNodeList.item(j).getAttributes().getNamedItem("href");
-                    Host foundHost;
-                    if (imageHref != null) {
-                        String imageUrl = imageHref.getTextContent().trim();
-                        logger.info(String.format("Scanning %s", imageUrl));
-                        foundHost = supportedHosts.stream().filter(host -> host.isSupported(imageUrl)).findFirst().orElse(null);
-                    } else {
-                        logger.warn("href is null, skipping");
-                        continue;
-                    }
-                    if (foundHost != null) {
-                        logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref));
-                        imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost));
-                    } else {
-                        logger.warn(String.format("unsupported host for %s, skipping", imageHref));
-                        continue;
-                    }
-                }
-            } catch (Exception e) {
-                throw new PostParseException(e);
-            }
-
-            if (!imagesList.isEmpty()) {
-                logger.info(String.format("Found %d images for post with id %s", imagesList.size(), postId));
-                posts.add(appStateService.createPost(postTitle, realUrl, imagesList, null, postId));
-            } else {
-                logger.warn(String.format("No images found for post with id %s, skipping", postId));
+                downloadQ.enqueue(post);
+            } catch (InterruptedException e) {
+                logger.warn("Interruption was caught");
+                Thread.currentThread().interrupt();
+                return;
             }
+            logger.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
+        } else {
+            post.setStatus(Post.Status.STOPPED);
+            logger.debug("Auto start downloads option is disabled");
         }
-        return posts;
+    }
+
+    public VRThreadParser createVRThreadParser(String threadId) {
+        return new VRThreadParser(threadId, cm, vipergirlsAuthService);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostParser.java

@@ -0,0 +1,215 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import tn.mnlr.vripper.entities.Image;
+import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.exception.DownloadException;
+import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.host.Host;
+
+import javax.xml.parsers.SAXParserFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static tn.mnlr.vripper.services.PostParser.VR_API;
+
+class VRPostParser {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRPostParser.class);
+
+    private static SAXParserFactory factory = SAXParserFactory.newInstance();
+
+    private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+            .handleIf(e -> e instanceof IOException)
+            .withBackoff(5, 30, ChronoUnit.SECONDS)
+            .withMaxRetries(4)
+            .abortOn(InterruptedException.class)
+            .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+
+    private final String threadId;
+    private final String postId;
+    private final ConnectionManager cm;
+    private final VipergirlsAuthService vipergirlsAuthService;
+    private final List<Host> supportedHosts;
+
+    VRPostParser(String threadId, String postId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> supportedHosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.cm = cm;
+        this.vipergirlsAuthService = vipergirlsAuthService;
+        this.supportedHosts = supportedHosts;
+    }
+
+    public Post parse() throws PostParseException {
+
+        logger.debug(String.format("Parsing post %s", postId));
+        HttpGet httpGet;
+        try {
+            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            uriBuilder.setParameter("p", postId);
+            httpGet = cm.buildHttpGet(uriBuilder.build());
+        } catch (URISyntaxException e) {
+            throw new PostParseException(e);
+        }
+        VRPostHandler handler = new VRPostHandler(threadId, postId, supportedHosts);
+        AtomicReference<Throwable> thr = new AtomicReference<>();
+        logger.debug(String.format("Requesting %s", httpGet));
+        Post post = Failsafe.with(retryPolicy)
+                .onFailure(e -> thr.set(e.getFailure()))
+                .get(() -> {
+                    HttpClient connection = cm.getClient().build();
+                    try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
+                        if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                            throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                        }
+
+                        try {
+                            factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                            return handler.getParsedPost();
+                        } catch (Exception e) {
+                            throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
+                        } finally {
+                            EntityUtils.consumeQuietly(response.getEntity());
+                        }
+                    }
+                });
+        if (thr.get() != null || post == null) {
+            logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
+            throw new PostParseException(thr.get());
+        }
+        return post;
+    }
+}
+
+@Getter
+class VRPostParseState extends VRPostState {
+
+    private final String type = "postParse";
+    private String postId;
+    private int number;
+    private String title;
+    private int imageCount;
+    private String url;
+    private List<String> previews;
+
+
+    VRPostParseState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews) {
+        super(threadId);
+        this.postId = postId;
+        this.number = number;
+        this.title = title;
+        this.imageCount = imageCount;
+        this.previews = previews;
+        this.url = url;
+    }
+}
+
+class VRPostHandler extends DefaultHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRPostHandler.class);
+
+    private final List<Host> supportedHosts;
+
+    private final String threadId;
+    private final String postId;
+    private String threadTitle;
+
+    private int previewCounter = 0;
+    private int index = 0;
+
+    private String postTitle;
+    private int imageCount;
+
+    private String forum;
+
+    private List<String> previews = new ArrayList<>();
+    private List<Image> images = new ArrayList<>();
+    @Getter
+    private Post parsedPost;
+
+    VRPostHandler(String threadId, String postId, List<Host> supportedHosts) {
+        this.threadId = threadId;
+        this.postId = postId;
+        this.supportedHosts = supportedHosts;
+    }
+
+    @Override
+    public void startDocument() {
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) {
+
+        switch (qName.toLowerCase()) {
+            case "forum":
+                forum = attributes.getValue("title").trim();
+                break;
+            case "thread":
+                threadTitle = attributes.getValue("title").trim();
+                break;
+            case "post":
+                imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
+                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                break;
+            case "image":
+                index++;
+                if (previewCounter++ < 4) {
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(thumbUrl -> previews.add(thumbUrl));
+                }
+
+                String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
+                if (mainUrl != null) {
+                    Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
+                    if (foundHost != null) {
+                        logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
+                        images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
+                    } else {
+                        logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
+                    }
+                }
+                break;
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName) {
+        if ("post".equals(qName.toLowerCase())) {
+            if (imageCount != 0) {
+                HashMap<String, Object> metadata = new HashMap<>();
+                metadata.put("PREVIEWS", previews);
+                parsedPost = new Post(
+                        postTitle,
+                        String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
+                        images,
+                        metadata,
+                        postId,
+                        threadId,
+                        threadTitle,
+                        forum
+                );
+            }
+            index = 0;
+            previewCounter = 0;
+            previews = new ArrayList<>();
+            images = new ArrayList<>();
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRPostState.java

@@ -0,0 +1,12 @@
+package tn.mnlr.vripper.services;
+
+import lombok.Getter;
+
+@Getter
+public abstract class VRPostState {
+    private final String threadId;
+
+    VRPostState(String threadId) {
+        this.threadId = threadId;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VRThreadParser.java

@@ -0,0 +1,176 @@
+package tn.mnlr.vripper.services;
+
+import io.reactivex.processors.PublishProcessor;
+import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import tn.mnlr.vripper.exception.DownloadException;
+import tn.mnlr.vripper.exception.PostParseException;
+
+import javax.xml.parsers.SAXParserFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static tn.mnlr.vripper.services.PostParser.VR_API;
+
+public class VRThreadParser {
+
+    private static final Logger logger = LoggerFactory.getLogger(VRThreadParser.class);
+
+    private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
+            .handleIf(e -> e instanceof IOException)
+            .withBackoff(5, 30, ChronoUnit.SECONDS)
+            .withMaxRetries(4)
+            .abortOn(InterruptedException.class)
+            .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+    private static SAXParserFactory factory = SAXParserFactory.newInstance();
+    @Getter
+    private final PublishProcessor<VRPostState> postPublishProcessor = PublishProcessor.create();
+    private final String threadId;
+    private final ConnectionManager cm;
+    private final VipergirlsAuthService vipergirlsAuthService;
+
+
+    VRThreadParser(String threadId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService) {
+        this.threadId = threadId;
+        this.cm = cm;
+        this.vipergirlsAuthService = vipergirlsAuthService;
+    }
+
+    public Void parse() throws PostParseException {
+
+        logger.debug(String.format("Parsing thread %s", threadId));
+        HttpGet httpGet;
+        try {
+            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            uriBuilder.setParameter("t", threadId);
+            httpGet = cm.buildHttpGet(uriBuilder.build());
+        } catch (URISyntaxException e) {
+            throw new PostParseException(e);
+        }
+
+        VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
+        AtomicReference<Throwable> thr = new AtomicReference<>();
+        logger.debug(String.format("Requesting %s", httpGet));
+        Failsafe.with(retryPolicy)
+                .onFailure(e -> thr.set(e.getFailure()))
+                .get(() -> {
+                    HttpClient connection = cm.getClient().build();
+                    try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
+                        if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                            throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                        }
+
+                        try {
+                            factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                        } catch (Exception e) {
+                            throw new PostParseException(String.format("Failed to parse thread %s", threadId), e);
+                        } finally {
+                            EntityUtils.consumeQuietly(response.getEntity());
+                        }
+                    }
+                    return null;
+                });
+        if (thr.get() != null) {
+            logger.error(String.format("parsing failed for thread %s", threadId), thr.get());
+            throw new PostParseException(thr.get());
+        }
+        return null;
+    }
+}
+
+class VRThreadHandler extends DefaultHandler {
+
+    private final String threadId;
+    private final PublishProcessor<VRPostState> vrPostPublishProcessor;
+    private String threadTitle;
+    private String postId;
+    private String postTitle;
+    private int imageCount;
+    private int postCounter;
+    private int previewCounter = 0;
+    private List<String> previews = new ArrayList<>();
+
+    VRThreadHandler(String threadId, PublishProcessor<VRPostState> vrPostPublishProcessor) {
+        this.vrPostPublishProcessor = vrPostPublishProcessor;
+        this.threadId = threadId;
+    }
+
+    @Override
+    public void startDocument() {
+        vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "START"));
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) {
+
+        switch (qName.toLowerCase()) {
+            case "thread":
+                threadTitle = attributes.getValue("title").trim();
+                break;
+            case "post":
+                imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
+                postId = attributes.getValue("id").trim();
+                postCounter = Integer.parseInt(attributes.getValue("number").trim());
+                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                break;
+            case "image":
+                if (previewCounter++ < 4) {
+                    Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(thumbUrl -> previews.add(thumbUrl));
+                }
+                break;
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName) {
+        if ("post".equals(qName.toLowerCase())) {
+            if (imageCount != 0) {
+                vrPostPublishProcessor.onNext(new VRPostParseState(
+                        threadId,
+                        postId,
+                        postCounter,
+                        postTitle,
+                        imageCount,
+                        String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
+                        previews
+                ));
+            }
+            previewCounter = 0;
+            previews = new ArrayList<>();
+        }
+    }
+
+    @Override
+    public void endDocument() {
+        vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "END"));
+        vrPostPublishProcessor.onComplete();
+    }
+}
+
+@Getter
+class VRThreadParseState extends VRPostState {
+
+    private final String type = "threadParseState";
+    private final String state;
+
+    VRThreadParseState(String threadId, String state) {
+        super(threadId);
+        this.state = state;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java

@@ -1,10 +1,15 @@
 package tn.mnlr.vripper.services;
 
+import io.reactivex.processors.PublishProcessor;
+import lombok.Getter;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
@@ -13,11 +18,15 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Document;
+import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.exception.VripperException;
 
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 @Service
 public class VipergirlsAuthService {
@@ -36,18 +45,39 @@ public class VipergirlsAuthService {
     @Autowired
     private AppSettingsService appSettingsService;
 
-    private String cookies;
+    @Getter
+    private HttpClientContext context = HttpClientContext.create();
 
     private boolean authenticated = false;
 
+    @Getter
+    private String loggedUser;
+
+    @Getter
+    private PublishProcessor<String> loggedInUser = PublishProcessor.create();
+
+    @PostConstruct
+    private void init() {
+        context.setCookieStore(new BasicCookieStore());
+    }
+
+    @PreDestroy
+    private void destroy() throws InterruptedException {
+        logger.info("Shutting down VipergirlsAuthService");
+        VripperApplication.commonExecutor.shutdown();
+        VripperApplication.commonExecutor.awaitTermination(10, TimeUnit.SECONDS);
+    }
+
     public void authenticate() throws VripperException {
 
         logger.info("Authenticating using ViperGirls credentials");
         authenticated = false;
 
         if(!appSettingsService.isVLogin()) {
-            logger.warn("Authentication option is disabled");
-            cookies = null;
+            logger.debug("Authentication option is disabled");
+            context.getCookieStore().clear();
+            loggedUser = "";
+            loggedInUser.onNext(loggedUser);
             return;
         }
 
@@ -56,6 +86,9 @@ public class VipergirlsAuthService {
 
         if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
             logger.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
+            context.getCookieStore().clear();
+            loggedUser = "";
+            loggedInUser.onNext(loggedUser);
             return;
         }
 
@@ -73,6 +106,9 @@ public class VipergirlsAuthService {
         try {
             postAuth.setEntity(new UrlEncodedFormEntity(params));
         } catch (Exception e) {
+            context.getCookieStore().clear();
+            loggedUser = "";
+            loggedInUser.onNext(loggedUser);
             throw new VripperException(e);
         }
 
@@ -81,63 +117,66 @@ public class VipergirlsAuthService {
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(postAuth)) {
+        try (CloseableHttpResponse response = client.execute(postAuth, context)) {
 
-            StringBuilder sb = new StringBuilder();
-            Arrays.asList(response.getHeaders("set-cookie"))
-                    .stream()
-                    .map(e -> e.getValue())
-                    .map(e -> e.split(";")[0])
-                    .filter(e -> e.startsWith("vg_lastactivity") || e.startsWith("vg_userid") || e.startsWith("vg_password"))
-                    .forEach(e -> sb.append(e).append(";"));
-            String cookies = sb.toString();
+            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                throw new VripperException(String.format("Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
+            }
             String responseBody = EntityUtils.toString(response.getEntity());
             logger.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
             EntityUtils.consumeQuietly(response.getEntity());
-            if(!cookies.contains("vg_userid")) {
+            if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) {
                 throw new VripperException("Failed to authenticate user with ViperRipper");
             }
-            this.cookies = cookies;
         } catch (Exception e) {
-            throw new VripperException(e);
+            context.getCookieStore().clear();
+            loggedUser = "";
+            loggedInUser.onNext(loggedUser);
+            if (e instanceof VripperException) {
+                throw (VripperException) e;
+            } else {
+                throw new VripperException(e);
+            }
         }
         authenticated = true;
+        loggedUser = username;
         logger.info(String.format("Authenticated: %s", username));
+        loggedInUser.onNext(loggedUser);
     }
 
-    public void leaveThanks(String postUrl, String postId) throws VripperException {
-
-        if (!appSettingsService.isVLogin()) {
-            logger.warn("Authentication with ViperGirls option is disabled");
-            return;
-        }
-        if (!appSettingsService.isVThanks()) {
-            logger.warn("Leave thanks option is disabled");
-            return;
-        }
-        if (!authenticated) {
-            logger.error("You are not authenticated");
-            return;
-        }
-        try {
-            postThanks(postId, getSecurityToken(postUrl));
-        } catch (VripperException e) {
-            throw new VripperException(e);
-        }
+    void leaveThanks(String postUrl, String postId) {
+        VripperApplication.commonExecutor.submit(() -> {
+            if (!appSettingsService.isVLogin()) {
+                logger.debug("Authentication with ViperGirls option is disabled");
+                return;
+            }
+            if (!appSettingsService.isVThanks()) {
+                logger.debug("Leave thanks option is disabled");
+                return;
+            }
+            if (!authenticated) {
+                logger.error("You are not authenticated");
+                return;
+            }
+            try {
+                postThanks(postId, getSecurityToken(postUrl));
+            } catch (Exception e) {
+                logger.error(String.format("Failed to leave a thanks for url %s, post id %s", postUrl, postId), e);
+            }
+        });
     }
 
     private String getSecurityToken(String url) throws VripperException {
 
-        String securityToken = "";
+        String securityToken;
 
         HttpGet httpGet = cm.buildHttpGet(url);
         httpGet.addHeader("Referer", "https://vipergirls.to/");
         httpGet.addHeader("Host", "vipergirls.to");
-        httpGet.addHeader("Cookie", cookies);
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(httpGet)) {
+        try (CloseableHttpResponse response = client.execute(httpGet, context)) {
 
             String postPage = EntityUtils.toString(response.getEntity());
             Document document = htmlProcessorService.clean(postPage);
@@ -149,7 +188,7 @@ public class VipergirlsAuthService {
                     .getTextContent()
                     .trim();
 
-            securityToken = Arrays.asList(thanksUrl.split("&amp;")).stream()
+            securityToken = Arrays.stream(thanksUrl.split("&amp;"))
                     .filter(v -> v.startsWith("securitytoken"))
                     .findAny()
                     .orElse("")
@@ -178,12 +217,10 @@ public class VipergirlsAuthService {
 
         postThanks.addHeader("Referer", "https://vipergirls.to/");
         postThanks.addHeader("Host", "vipergirls.to");
-        postThanks.addHeader("Cookie", cookies);
 
         CloseableHttpClient client = cm.getClient().build();
 
-        try (CloseableHttpResponse response = client.execute(postThanks)) {
-
+        try (CloseableHttpResponse response = client.execute(postThanks, context)) {
             EntityUtils.consumeQuietly(response.getEntity());
         } catch (Exception e) {
             throw new VripperException(e);

vripper-server/src/main/java/tn/mnlr/vripper/services/XpathService.java

@@ -2,7 +2,6 @@ package tn.mnlr.vripper.services;
 
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 import tn.mnlr.vripper.exception.XpathException;
 
 import javax.xml.xpath.XPath;
@@ -12,20 +11,8 @@ import javax.xml.xpath.XPathFactory;
 @Service
 public class XpathService {
 
-    private XpathService() {
-
-    }
-
     private XPath xpath = XPathFactory.newInstance().newXPath();
 
-    public NodeList getAsNodeList(Node source, String xpathExpression) throws XpathException {
-        try {
-            return (NodeList) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET);
-        } catch (Exception e) {
-            throw new XpathException(e);
-        }
-    }
-
     public Node getAsNode(Node source, String xpathExpression) throws XpathException {
         try {
             return (Node) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE);

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java

@@ -1,21 +1,28 @@
 package tn.mnlr.vripper.web.restendpoints;
 
 import lombok.Getter;
-import lombok.ToString;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import tn.mnlr.vripper.entities.Post;
+import tn.mnlr.vripper.VripperApplication;
+import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.q.DownloadQ;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.AppStateService;
+import tn.mnlr.vripper.services.PathService;
 import tn.mnlr.vripper.services.PostParser;
-import tn.mnlr.vripper.services.VipergirlsAuthService;
 
+import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 @RestController
 @CrossOrigin(value = "*")
@@ -23,6 +30,8 @@ public class PostRestEndpoint {
 
     private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
 
+    private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?");
+
     @Autowired
     private AppStateService appStateService;
 
@@ -30,7 +39,7 @@ public class PostRestEndpoint {
     private AppSettingsService appSettingsService;
 
     @Autowired
-    private VipergirlsAuthService authService;
+    private PathService pathService;
 
     @ExceptionHandler(Exception.class)
     public ResponseEntity handleException(Exception e) {
@@ -49,59 +58,87 @@ public class PostRestEndpoint {
     @PostMapping("/post")
     @ResponseStatus(value = HttpStatus.OK)
     public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception {
-        logger.info(String.format("Starting to process thread: %s", url.url));
-        if (url.url == null || url.url.isEmpty()) {
+        logger.debug(String.format("Starting to process thread: %s", url.getUrl()));
+        if (url.getUrl() == null || url.getUrl().isEmpty()) {
             return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST);
-        } else if (!url.url.startsWith("https://vipergirls.to")) {
+        } else if (!url.getUrl().startsWith("https://vipergirls.to")) {
             return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
         }
-        List<Post> parsed = postParser.parse(url.url);
-        logger.info(String.format("%d posts found from thread %s", parsed.size(), url.url));
-        parsed.forEach(p -> {
-            try {
-                authService.leaveThanks(p.getUrl(), p.getPostId());
-            } catch (Exception e) {
-                logger.error(String.format("Failed to leave thanks for %s", p.getUrl()), e);
-            }
-        });
-        if(appSettingsService.isAutoStart()) {
-            logger.info("Auto start downloads option is enabled");
-            logger.info(String.format("Starting to enqueue %d jobs for %s", parsed.stream().flatMap(e -> e.getImages().stream()).count(), url.url));
-            for (Post post : parsed) {
-                downloadQ.enqueue(post);
-            }
-            logger.info(String.format("Done enqueuing jobs for %s", url.url));
-        } else {
-            logger.info("Auto start downloads option is disabled");
-        }
-        logger.info(String.format("Done processing thread: %s", url.url));
-        return ResponseEntity.ok(new ParseResult(parsed.size()));
-    }
 
-    @PostMapping("/clipboard/post")
-    @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity processPostFromClipboard(@RequestBody ThreadUrl url) throws Exception {
-        return this.processPost(url);
+        String threadId, postId;
+        try {
+            Matcher m = VG_URL_PATTERN.matcher(url.getUrl());
+            if (m.find()) {
+                threadId = m.group(1);
+                postId = m.group(4);
+            } else {
+                throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url));
+            }
+        } catch (Exception e) {
+            throw new PostParseException(String.format("Cannot retrieve thread id from URL %s", url), e);
+        }
+        return ResponseEntity.ok(new PairThreadIdPostId(threadId, postId));
     }
 
     @PostMapping("/post/restart")
     @ResponseStatus(value = HttpStatus.OK)
-    public void restartPost(@RequestBody PostId postId) throws Exception {
-        downloadQ.restart(postId.getPostId());
+    public synchronized void restartPost(@RequestBody @NonNull List<PostId> postIds) throws Exception {
+        for (PostId postId : postIds) {
+            downloadQ.restart(postId.getPostId());
+        }
+    }
+
+    @PostMapping("/post/add")
+    @ResponseStatus(value = HttpStatus.OK)
+    public void addPost(@RequestBody List<PostToAdd> posts) {
+        for (PostToAdd post : posts) {
+            VripperApplication.commonExecutor.submit(() -> {
+                try {
+                    postParser.addPost(post.getPostId(), post.getThreadId());
+                } catch (PostParseException e) {
+                    logger.error(String.format("Failed to add post %s", post.getPostId()), e);
+                }
+            });
+        }
+    }
+
+    @GetMapping("/post/path/{postId}")
+    @ResponseStatus(value = HttpStatus.OK)
+    public ResponseEntity<DownloadPath> folderPath(@PathVariable("postId") String postId) {
+        File destinationFolder = pathService.getDownloadDestinationFolder(postId);
+        return ResponseEntity.ok(new DownloadPath(destinationFolder.getPath()));
+    }
+
+    @PostMapping("/post/restart/all")
+    @ResponseStatus(value = HttpStatus.OK)
+    public void restartPost() throws Exception {
+        downloadQ.restartAll();
     }
 
     @PostMapping("/post/stop")
     @ResponseStatus(value = HttpStatus.OK)
-    public void stop(@RequestBody PostId postId) {
-        downloadQ.stop(postId.getPostId());
+    public synchronized void stop(@RequestBody @NonNull List<PostId> postIds) {
+        for (PostId postId : postIds) {
+            downloadQ.stop(postId.getPostId());
+        }
+    }
+
+    @PostMapping("/post/stop/all")
+    @ResponseStatus(value = HttpStatus.OK)
+    public void stopAll() {
+        downloadQ.stopAll();
     }
 
     @PostMapping("/post/remove")
     @ResponseStatus(value = HttpStatus.OK)
-    public ResponseEntity remove(@RequestBody PostId postId) {
-        downloadQ.stop(postId.getPostId());
-        appStateService.remove(postId.getPostId());
-        return ResponseEntity.ok(new RemoveResult(postId.getPostId()));
+    public synchronized ResponseEntity remove(@RequestBody @NonNull List<PostId> postIds) {
+        List<RemoveResult> result = new ArrayList<>();
+        for (PostId postId : postIds) {
+            downloadQ.stop(postId.getPostId());
+            appStateService.remove(postId.getPostId());
+            result.add(new RemoveResult(postId.getPostId()));
+        }
+        return ResponseEntity.ok(result);
     }
 
     @PostMapping("/post/clear/all")
@@ -115,43 +152,87 @@ public class PostRestEndpoint {
     public ResponseEntity removeAll() {
         return ResponseEntity.ok(new RemoveAllResult(appStateService.removeAll()));
     }
+}
 
-    @Getter
-    @ToString
-    private static class ThreadUrl {
-        private String url;
-    }
+@Getter
+@Setter
+@NoArgsConstructor
+class ThreadUrl {
+    private String url;
 
-    @Getter
-    private static class PostId {
-        private String postId;
-    }
-
-    @Getter
-    private static class ParseResult {
-        ParseResult(int parsed) {
-            this.parsed = parsed;
-        }
-        private int parsed;
-    }
-
-    @Getter
-    private static class RemoveAllResult {
-        private List<String> postIds;
-        private int removed;
-
-        RemoveAllResult(List<String> postIds) {
-            this.removed = postIds.size();
-            this.postIds = postIds;
-        }
-    }
-
-    @Getter
-    private static class RemoveResult {
-        private String postId;
-
-        RemoveResult(String postId) {
-            this.postId = postId;
-        }
+    public ThreadUrl(String url) {
+        this.url = url;
     }
 }
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PostId {
+    private String postId;
+
+    public PostId(String postId) {
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PairThreadIdPostId {
+    private String threadId;
+    private String postId;
+
+    PairThreadIdPostId(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class PostToAdd {
+    private String threadId;
+    private String postId;
+
+    public PostToAdd(String threadId, String postId) {
+        this.threadId = threadId;
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class RemoveAllResult {
+    private List<String> postIds;
+    private int removed;
+
+    RemoveAllResult(List<String> postIds) {
+        this.removed = postIds.size();
+        this.postIds = postIds;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class RemoveResult {
+    private String postId;
+
+    RemoveResult(String postId) {
+        this.postId = postId;
+    }
+}
+
+@Getter
+@Setter
+@NoArgsConstructor
+class DownloadPath {
+    private String path;
+
+    DownloadPath(String path) {
+        this.path = path;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -7,8 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.exception.ValidationException;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.VipergirlsAuthService;
 
 @RestController
@@ -31,42 +31,65 @@ public class SettingsRestEndpoint {
                 .body(e.getMessage());
     }
 
+    @PostMapping("/settings/theme")
+    @ResponseStatus(value = HttpStatus.OK)
+    public AppSettingsService.Theme postTheme(@RequestBody AppSettingsService.Theme theme) {
+        synchronized (this.settings) {
+            this.settings.setTheme(theme);
+            return settings.getTheme();
+        }
+    }
+
+    @GetMapping("/settings/theme")
+    @ResponseStatus(value = HttpStatus.OK)
+    public AppSettingsService.Theme getTheme() {
+        return settings.getTheme();
+    }
+
     @PostMapping("/settings")
     @ResponseStatus(value = HttpStatus.OK)
     public ResponseEntity postSettings(@RequestBody AppSettingsService.Settings settings) throws Exception {
 
-        try {
-            this.settings.check(settings);
-        } catch (ValidationException e) {
-            return new ResponseEntity(new Response(e.getMessage()), HttpStatus.BAD_REQUEST);
-        }
-        this.settings.setDownloadPath(settings.getDownloadPath());
-        this.settings.setMaxThreads(settings.getMaxThreads());
-        this.settings.setAutoStart(settings.isAutoStart());
-        this.settings.setVLogin(settings.isVLogin());
-
-        if(settings.isVLogin()) {
-            this.settings.setVUsername(settings.getVUsername());
-            if (!this.settings.getVPassword().equals(settings.getVPassword())) {
-                this.settings.setVPassword(settings.getVPassword());
+        synchronized (this.settings) {
+            try {
+                this.settings.check(settings);
+            } catch (ValidationException e) {
+                return ResponseEntity
+                        .status(HttpStatus.BAD_REQUEST)
+                        .body(new Response(e.getMessage()));
             }
-            this.settings.setVThanks(settings.isVThanks());
-        } else {
-            this.settings.setVUsername("");
-            this.settings.setVPassword("");
-            this.settings.setVThanks(false);
+            this.settings.setDownloadPath(settings.getDownloadPath());
+            this.settings.setMaxThreads(settings.getMaxThreads());
+            this.settings.setAutoStart(settings.isAutoStart());
+            this.settings.setVLogin(settings.isVLogin());
+
+            if (settings.isVLogin()) {
+                this.settings.setVUsername(settings.getVUsername());
+                if (!this.settings.getVPassword().equals(settings.getVPassword())) {
+                    this.settings.setVPassword(settings.getVPassword());
+                }
+                this.settings.setVThanks(settings.isVThanks());
+            } else {
+                this.settings.setVUsername("");
+                this.settings.setVPassword("");
+                this.settings.setVThanks(false);
+            }
+            this.settings.setDesktopClipboard(settings.isDesktopClipboard());
+            this.settings.setForceOrder(settings.isForceOrder());
+            this.settings.setSubLocation(settings.isSubLocation());
+            this.settings.setThreadSubLocation(settings.isThreadSubLocation());
+            this.settings.setClearCompleted(settings.isClearCompleted());
+
+            this.settings.save();
+
+            vipergirlsAuthService.authenticate();
         }
-        this.settings.setDesktopClipboard(settings.isDesktopClipboard());
-
-        this.settings.save();
-
-        vipergirlsAuthService.authenticate();
         return ResponseEntity.ok(getSettings());
     }
 
     @GetMapping("/settings")
     @ResponseStatus(value = HttpStatus.OK)
-    public AppSettingsService.Settings getSettings() throws Exception {
+    public AppSettingsService.Settings getSettings() {
 
         return new AppSettingsService.Settings(
                 settings.getDownloadPath(),
@@ -76,15 +99,19 @@ public class SettingsRestEndpoint {
                 settings.getVUsername(),
                 settings.getVPassword(),
                 settings.isVThanks(),
-                settings.isDesktopClipboard()
+                settings.isDesktopClipboard(),
+                settings.isForceOrder(),
+                settings.isSubLocation(),
+                settings.isThreadSubLocation(),
+                settings.isClearCompleted()
         );
     }
 
     @Getter
-    class Response {
+    static class Response {
         String message;
 
-        public Response(String message) {
+        Response(String message) {
             this.message = message;
         }
     }

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java

@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import io.reactivex.disposables.Disposable;
 import io.reactivex.schedulers.Schedulers;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,16 +14,19 @@ import org.springframework.web.socket.CloseStatus;
 import org.springframework.web.socket.TextMessage;
 import org.springframework.web.socket.WebSocketSession;
 import org.springframework.web.socket.handler.TextWebSocketHandler;
+import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.entities.Image;
 import tn.mnlr.vripper.entities.Post;
 import tn.mnlr.vripper.entities.mixin.ui.ImageUIMixin;
 import tn.mnlr.vripper.entities.mixin.ui.PostUIMixin;
-import tn.mnlr.vripper.services.AppStateService;
+import tn.mnlr.vripper.services.*;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -30,57 +35,168 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     private static final Logger logger = LoggerFactory.getLogger(WebSocketHandler.class);
 
-    @Getter
-    private static class WSMessage {
-
-        private String cmd;
-        private String payload;
-
-        enum CMD {
-            POSTS_SUB,
-            POST_DETAILS_SUB,
-            POSTS_UNSUB,
-            POST_DETAILS_UNSUB
-        }
-    }
-
     public WebSocketHandler() {
         om.addMixIn(Image.class, ImageUIMixin.class).addMixIn(Post.class, PostUIMixin.class);
     }
 
-    private Map<String, Disposable> postsSubscriptions = new ConcurrentHashMap<>();
-    private Map<String, Disposable> postDetailsSubscriptions = new ConcurrentHashMap<>();
-    private ObjectMapper om = new ObjectMapper();
+    @Autowired
+    private GlobalStateService globalStateService;
 
     @Autowired
     private AppStateService appStateService;
 
+    @Autowired
+    private DownloadSpeedService downloadSpeedService;
+
+    @Autowired
+    private VipergirlsAuthService vipergirlsAuthService;
+
+    @Autowired
+    private PostParser postParser;
+
+    private Map<String, Disposable> postsSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> postDetailsSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> vrPostParserSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> stateSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> downloadSpeedSubscriptions = new ConcurrentHashMap<>();
+    private Map<String, Disposable> userSubscriptions = new ConcurrentHashMap<>();
+
+    private Map<String, Future<Void>> threadParseRequests = new ConcurrentHashMap<>();
+
+    private ObjectMapper om = new ObjectMapper();
+
     @Override
     protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception {
 
         WSMessage wsMessage = om.readValue(message.getPayload(), WSMessage.class);
         WSMessage.CMD cmd = WSMessage.CMD.valueOf(wsMessage.getCmd());
         switch (cmd) {
+            case GLOBAL_STATE_SUB:
+                subscribeForGlobalState(session);
+                break;
+            case SPEED_SUB:
+                subscribeForSpeed(session);
+                break;
+            case USER_SUB:
+                subscribeForUser(session);
+                break;
             case POSTS_SUB:
                 subscribeForPosts(session);
                 break;
+            case THREAD_PARSING_SUB:
+                subscribeForThreadParsing(session, wsMessage.getPayload());
+                break;
             case POST_DETAILS_SUB:
                 subscribeForPostDetails(session, wsMessage.getPayload());
                 break;
             case POST_DETAILS_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from post details", session.getId()));
-                Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from post details", session.getId()));
+                Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
+            case THREAD_PARSING_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from thread parsing", session.getId()));
+                Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
                 break;
             case POSTS_UNSUB:
-                logger.info(String.format("Client %s unsubscribed from posts", session.getId()));
-                Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+                logger.debug(String.format("Client %s unsubscribed from posts", session.getId()));
+                Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
+            case GLOBAL_STATE_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from global state", session.getId()));
+                Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
+            case SPEED_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from download speed info", session.getId()));
+                Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+                break;
+            case USER_UNSUB:
+                logger.debug(String.format("Client %s unsubscribed from user info", session.getId()));
+                Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
                 break;
         }
     }
 
+    private void subscribeForGlobalState(WebSocketSession session) {
+
+        logger.debug(String.format("Client %s subscribed for global state", session.getId()));
+        if (stateSubscriptions.containsKey(session.getId())) {
+            stateSubscriptions.get(session.getId()).dispose();
+        }
+
+        try {
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(globalStateService.getCurrentState()))));
+        } catch (Exception e) {
+            logger.error("Unexpected error occurred", e);
+        }
+
+        stateSubscriptions.put(session.getId(),
+                globalStateService.getLiveGlobalState()
+                        .onBackpressureBuffer()
+                        .observeOn(Schedulers.io())
+                        .buffer(5000, TimeUnit.MILLISECONDS)
+                        .filter(e -> !e.isEmpty())
+                        .map(e -> e.subList(e.size() - 1, e.size()))
+                        .map(om::writeValueAsString)
+                        .map(TextMessage::new)
+                        .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        );
+    }
+
+    private void subscribeForSpeed(WebSocketSession session) {
+
+        logger.debug(String.format("Client %s subscribed for download speed info", session.getId()));
+        if (downloadSpeedSubscriptions.containsKey(session.getId())) {
+            downloadSpeedSubscriptions.get(session.getId()).dispose();
+        }
+
+        try {
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new DownloadSpeed(0)))));
+        } catch (Exception e) {
+            logger.error("Unexpected error occurred", e);
+        }
+
+        downloadSpeedSubscriptions.put(session.getId(),
+                downloadSpeedService.getReadBytesPerSecond()
+                        .onBackpressureBuffer()
+                        .observeOn(Schedulers.io())
+                        .buffer(5000, TimeUnit.MILLISECONDS)
+                        .filter(e -> !e.isEmpty())
+                        .map(e -> e.subList(e.size() - 1, e.size()))
+                        .map(e -> e.stream().map(DownloadSpeed::new).collect(Collectors.toList()))
+                        .map(om::writeValueAsString)
+                        .map(TextMessage::new)
+                        .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        );
+    }
+
+    private void subscribeForUser(WebSocketSession session) {
+
+        logger.debug(String.format("Client %s subscribed for user info", session.getId()));
+        if (userSubscriptions.containsKey(session.getId())) {
+            userSubscriptions.get(session.getId()).dispose();
+        }
+
+        try {
+            send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
+        } catch (Exception e) {
+            logger.error("Unexpected error occurred", e);
+        }
+
+        userSubscriptions.put(session.getId(),
+                vipergirlsAuthService.getLoggedInUser()
+                        .onBackpressureBuffer()
+                        .observeOn(Schedulers.io())
+                        .map(e -> Collections.singletonList(new LoggedUser(e)))
+                        .map(om::writeValueAsString)
+                        .map(TextMessage::new)
+                        .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        );
+    }
+
     private void subscribeForPosts(WebSocketSession session) {
 
-        logger.info(String.format("Client %s subscribed for posts", session.getId()));
+        logger.debug(String.format("Client %s subscribed for posts", session.getId()));
         if (postsSubscriptions.containsKey(session.getId())) {
             postsSubscriptions.get(session.getId()).dispose();
         }
@@ -95,8 +211,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 appStateService.getLivePostsState()
                         .onBackpressureBuffer()
                         .observeOn(Schedulers.io())
-                        .filter(e -> !e.isRemoved())
-                        .buffer(500, TimeUnit.MILLISECONDS, 50)
+                        .buffer(2000, TimeUnit.MILLISECONDS, 200)
                         .filter(e -> !e.isEmpty())
                         .map(e -> e.stream().distinct().collect(Collectors.toList()))
                         .map(om::writeValueAsString)
@@ -105,9 +220,35 @@ public class WebSocketHandler extends TextWebSocketHandler {
         );
     }
 
+    private void subscribeForThreadParsing(WebSocketSession session, String threadId) {
+
+        logger.debug(String.format("Client %s subscribed for thread parsing with threadId = %s", session.getId(), threadId));
+        if (vrPostParserSubscriptions.containsKey(session.getId())) {
+            vrPostParserSubscriptions.get(session.getId()).dispose();
+        }
+
+        if (threadParseRequests.containsKey(session.getId())) {
+            threadParseRequests.get(session.getId()).cancel(true);
+        }
+
+        VRThreadParser vrThreadParser = postParser.createVRThreadParser(threadId);
+
+        vrPostParserSubscriptions.put(session.getId(), vrThreadParser.getPostPublishProcessor()
+                .onBackpressureBuffer()
+                .observeOn(Schedulers.io())
+                .buffer(200)
+                .filter(e -> !e.isEmpty())
+                .map(om::writeValueAsString)
+                .map(TextMessage::new)
+                .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
+        );
+
+        threadParseRequests.put(session.getId(), VripperApplication.commonExecutor.submit(vrThreadParser::parse));
+    }
+
     private void subscribeForPostDetails(WebSocketSession session, String postId) {
 
-        logger.info(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
+        logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
         if (postDetailsSubscriptions.containsKey(session.getId())) {
             postDetailsSubscriptions.get(session.getId()).dispose();
         }
@@ -128,7 +269,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
                 .onBackpressureBuffer()
                 .observeOn(Schedulers.io())
                 .filter(e -> e.getPostId().equals(postId))
-                .buffer(500, TimeUnit.MILLISECONDS, 50)
+                .buffer(2000, TimeUnit.MILLISECONDS, 500)
                 .filter(e -> !e.isEmpty())
                 .map(e -> e.stream().distinct().collect(Collectors.toList()))
                 .map(om::writeValueAsString)
@@ -143,13 +284,55 @@ public class WebSocketHandler extends TextWebSocketHandler {
 
     @Override
     public void afterConnectionEstablished(WebSocketSession session) {
-
+        logger.debug(String.format("Connection open for client id: %s", session.getId()));
     }
 
     @Override
     public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
 
-        Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
-        Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
+        Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
+        Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
+
+        logger.debug(String.format("Connection closed for client id: %s", session.getId()));
+    }
+
+    @Getter
+    @Setter
+    @NoArgsConstructor
+    private static class WSMessage {
+
+        private String cmd;
+        private String payload;
+
+        enum CMD {
+            POSTS_SUB,
+            POST_DETAILS_SUB,
+            POSTS_UNSUB,
+            POST_DETAILS_UNSUB,
+            GLOBAL_STATE_SUB,
+            GLOBAL_STATE_UNSUB,
+            SPEED_SUB,
+            SPEED_UNSUB,
+            THREAD_PARSING_SUB,
+            THREAD_PARSING_UNSUB,
+            USER_SUB,
+            USER_UNSUB
+        }
+    }
+
+    @Getter
+    private static class LoggedUser {
+
+        private final String type = "user";
+        private String user;
+
+        LoggedUser(String user) {
+            this.user = user;
+        }
     }
 }

vripper-server/src/main/resources/application-installer.properties

@@ -0,0 +1 @@
+base.dir=${user.home}

vripper-server/src/main/resources/application-portable.properties

@@ -0,0 +1 @@
+base.dir=${user.dir}

vripper-server/src/main/resources/application.properties

@@ -1,5 +1,10 @@
 logging.level.org.springframework.web=INFO
 logging.level.root=INFO
 logging.level.org.apache.http=INFO
-logging.file=${user.dir}/vripper.log
-server.port=${vripper.server.port:8080}
+logging.file=${base.dir}/.vripper/vripper.log
+server.port=${vripper.server.port:8080}
+management.endpoints.web.exposure.include=shutdown
+management.endpoint.shutdown.enabled=true
+endpoints.shutdown.enabled=true
+spring.profiles.active=portable
+#spring.profiles.active=installer

vripper-server/src/main/resources/logback-spring.xml

@@ -7,8 +7,17 @@
 
     <include resource=
                      "org/springframework/boot/logging/logback/console-appender.xml" />
-    <root level="INFO">
+
+    <appender name="ASYNC-FILE" class="ch.qos.logback.classic.AsyncAppender">
         <appender-ref ref="FILE" />
+    </appender>
+
+    <appender name="ASYNC-CONSOLE" class="ch.qos.logback.classic.AsyncAppender">
         <appender-ref ref="CONSOLE" />
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="ASYNC-FILE"/>
+        <appender-ref ref="ASYNC-CONSOLE"/>
     </root>
 </configuration>

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "1.4.3",
+  "version": "2.3.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -1588,14 +1588,14 @@
       "dev": true
     },
     "ag-grid-angular": {
-      "version": "21.0.1",
-      "resolved": "https://registry.npmjs.org/ag-grid-angular/-/ag-grid-angular-21.0.1.tgz",
-      "integrity": "sha512-o0mhoPXbZDlxLAlwK+LPpBm5/RA6bYHqXV26yfJmx2wZ/khtT7VOiMbe9XAsGskMv+sxuR7G2FkxbxhluZzTnA=="
+      "version": "21.2.1",
+      "resolved": "https://registry.npmjs.org/ag-grid-angular/-/ag-grid-angular-21.2.1.tgz",
+      "integrity": "sha512-4bJ7+Gv6AY+MEJACGD16ZKDhQnBrU7wIaXsAwGbShb8jt8xDRhaS9Hf1jnEhY7vaLfYj+zR9vrXGXbTvwTtY2Q=="
     },
     "ag-grid-community": {
-      "version": "21.0.1",
-      "resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-21.0.1.tgz",
-      "integrity": "sha512-wqm6W39kImPonjRxHvnt/L6roJLxPXSopXF4pvcE1paxN7S25HNsBgTOB1uaLlQ4/xt3xSpccPbTDqVUnSxrmQ=="
+      "version": "21.2.1",
+      "resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-21.2.1.tgz",
+      "integrity": "sha512-UmS0j7pOz6SqN++Thg1a82LghW1zKk0k30kpPWxlOzC5On5b8MyKVsYZbGFtpdIHyJlPH3qbkw+PcFWSK168sw=="
     },
     "agent-base": {
       "version": "4.2.1",
@@ -1759,12 +1759,6 @@
       "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==",
       "dev": true
     },
-    "array-slice": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-0.2.3.tgz",
-      "integrity": "sha1-3Tz7gO15c6dRF82sabC5nshhhvU=",
-      "dev": true
-    },
     "array-union": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz",
@@ -2226,31 +2220,59 @@
       "dev": true
     },
     "body-parser": {
-      "version": "1.18.3",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz",
-      "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=",
+      "version": "1.19.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
+      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
       "dev": true,
       "requires": {
-        "bytes": "3.0.0",
+        "bytes": "3.1.0",
         "content-type": "~1.0.4",
         "debug": "2.6.9",
         "depd": "~1.1.2",
-        "http-errors": "~1.6.3",
-        "iconv-lite": "0.4.23",
+        "http-errors": "1.7.2",
+        "iconv-lite": "0.4.24",
         "on-finished": "~2.3.0",
-        "qs": "6.5.2",
-        "raw-body": "2.3.3",
-        "type-is": "~1.6.16"
+        "qs": "6.7.0",
+        "raw-body": "2.4.0",
+        "type-is": "~1.6.17"
       },
       "dependencies": {
-        "iconv-lite": {
-          "version": "0.4.23",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
-          "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==",
+        "bytes": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
+          "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==",
+          "dev": true
+        },
+        "http-errors": {
+          "version": "1.7.2",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
+          "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
           "dev": true,
           "requires": {
-            "safer-buffer": ">= 2.1.2 < 3"
+            "depd": "~1.1.2",
+            "inherits": "2.0.3",
+            "setprototypeof": "1.1.1",
+            "statuses": ">= 1.5.0 < 2",
+            "toidentifier": "1.0.0"
           }
+        },
+        "qs": {
+          "version": "6.7.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
+          "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==",
+          "dev": true
+        },
+        "setprototypeof": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
+          "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==",
+          "dev": true
+        },
+        "statuses": {
+          "version": "1.5.0",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+          "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=",
+          "dev": true
         }
       }
     },
@@ -2637,12 +2659,6 @@
       "integrity": "sha512-oC7/DVAyfcY3UWKm0sN/oVoDedQDQiw/vIiAnuTWTpE5s0zWf7l3WY417Xw/Fbi/QbAjctAkxgMiS9P0s3zkmA==",
       "dev": true
     },
-    "circular-json": {
-      "version": "0.5.9",
-      "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.5.9.tgz",
-      "integrity": "sha512-4ivwqHpIFJZBuhN3g/pEcdbnGUywkBblloGbkglyloVjjR3uT6tieI89MVOfbP2tHX5sgb01FuLgAOzebNlJNQ==",
-      "dev": true
-    },
     "class-utils": {
       "version": "0.3.6",
       "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
@@ -2798,15 +2814,6 @@
       "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=",
       "dev": true
     },
-    "combine-lists": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/combine-lists/-/combine-lists-1.0.1.tgz",
-      "integrity": "sha1-RYwH4J4NkA/Ci3Cj/sLazR0st/Y=",
-      "dev": true,
-      "requires": {
-        "lodash": "^4.5.0"
-      }
-    },
     "combined-stream": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
@@ -2903,36 +2910,21 @@
       }
     },
     "connect": {
-      "version": "3.6.6",
-      "resolved": "https://registry.npmjs.org/connect/-/connect-3.6.6.tgz",
-      "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=",
+      "version": "3.7.0",
+      "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz",
+      "integrity": "sha512-ZqRXc+tZukToSNmh5C2iWMSoV3X1YUcPbqEM4DkEG5tNQXrQUZCNVGGv3IuicnkMtPfGf3Xtp8WCXs295iQ1pQ==",
       "dev": true,
       "requires": {
         "debug": "2.6.9",
-        "finalhandler": "1.1.0",
-        "parseurl": "~1.3.2",
+        "finalhandler": "1.1.2",
+        "parseurl": "~1.3.3",
         "utils-merge": "1.0.1"
       },
       "dependencies": {
-        "finalhandler": {
-          "version": "1.1.0",
-          "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz",
-          "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=",
-          "dev": true,
-          "requires": {
-            "debug": "2.6.9",
-            "encodeurl": "~1.0.1",
-            "escape-html": "~1.0.3",
-            "on-finished": "~2.3.0",
-            "parseurl": "~1.3.2",
-            "statuses": "~1.3.1",
-            "unpipe": "~1.0.0"
-          }
-        },
-        "statuses": {
-          "version": "1.3.1",
-          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz",
-          "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=",
+        "parseurl": {
+          "version": "1.3.3",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
+          "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
           "dev": true
         }
       }
@@ -3202,9 +3194,9 @@
       }
     },
     "date-format": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/date-format/-/date-format-1.2.0.tgz",
-      "integrity": "sha1-YV6CjiM90aubua4JUODOzPpuytg=",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/date-format/-/date-format-2.1.0.tgz",
+      "integrity": "sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==",
       "dev": true
     },
     "date-now": {
@@ -3518,9 +3510,9 @@
       "dev": true
     },
     "electron": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/electron/-/electron-5.0.5.tgz",
-      "integrity": "sha512-GzVQhImBX3rSCFPyJ1u1KbxquoidAHzGeCH2FTs3lzAh1H8m4vd7xh6CNC111mT/I8pxFk5D8s3atJlJQLPAeg==",
+      "version": "6.0.9",
+      "resolved": "https://registry.npmjs.org/electron/-/electron-6.0.9.tgz",
+      "integrity": "sha512-lFpSmDNyjpvJFwEnK897Soone3DV7D3ASFUb315H2VTVZSbKib9Kbrsovxf4c+e1q5hTdaONsGIm3Lb4CXIW1g==",
       "dev": true,
       "requires": {
         "@types/node": "^10.12.18",
@@ -3529,9 +3521,9 @@
       },
       "dependencies": {
         "@types/node": {
-          "version": "10.14.10",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.10.tgz",
-          "integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q==",
+          "version": "10.14.18",
+          "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.18.tgz",
+          "integrity": "sha512-ryO3Q3++yZC/+b8j8BdKd/dn9JlzlHBPdm80656xwYUdmPkpTGTjkAdt6BByiNupGPE8w0FhBgvYy/fX9hRNGQ==",
           "dev": true
         }
       }
@@ -3879,56 +3871,6 @@
       "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=",
       "dev": true
     },
-    "expand-braces": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/expand-braces/-/expand-braces-0.1.2.tgz",
-      "integrity": "sha1-SIsdHSRRyz06axks/AMPRMWFX+o=",
-      "dev": true,
-      "requires": {
-        "array-slice": "^0.2.3",
-        "array-unique": "^0.2.1",
-        "braces": "^0.1.2"
-      },
-      "dependencies": {
-        "array-unique": {
-          "version": "0.2.1",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.2.1.tgz",
-          "integrity": "sha1-odl8yvy8JiXMcPrc6zalDFiwGlM=",
-          "dev": true
-        },
-        "braces": {
-          "version": "0.1.5",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-0.1.5.tgz",
-          "integrity": "sha1-wIVxEIUpHYt1/ddOqw+FlygHEeY=",
-          "dev": true,
-          "requires": {
-            "expand-range": "^0.1.0"
-          }
-        },
-        "expand-range": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-0.1.1.tgz",
-          "integrity": "sha1-TLjtoJk8pW+k9B/ELzy7TMrf8EQ=",
-          "dev": true,
-          "requires": {
-            "is-number": "^0.1.1",
-            "repeat-string": "^0.2.2"
-          }
-        },
-        "is-number": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-0.1.1.tgz",
-          "integrity": "sha1-aaevEWlj1HIG7JvZtIoUIW8eOAY=",
-          "dev": true
-        },
-        "repeat-string": {
-          "version": "0.2.2",
-          "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-0.2.2.tgz",
-          "integrity": "sha1-x6jTI2BoNiBZp+RlH8aITosftK4=",
-          "dev": true
-        }
-      }
-    },
     "expand-brackets": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
@@ -4401,9 +4343,9 @@
       }
     },
     "flatted": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz",
-      "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
+      "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==",
       "dev": true
     },
     "flush-write-stream": {
@@ -6075,28 +6017,27 @@
       }
     },
     "karma": {
-      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/karma/-/karma-3.1.4.tgz",
-      "integrity": "sha512-31Vo8Qr5glN+dZEVIpnPCxEGleqE0EY6CtC2X9TagRV3rRQ3SNrvfhddICkJgUK3AgqpeKSZau03QumTGhGoSw==",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/karma/-/karma-4.2.0.tgz",
+      "integrity": "sha512-fmCuxN1rwJxTdZfOXK5LjlmS4Ana/OvzNMpkyLL/TLE8hmgSkpVpMYQ7RTVa8TNKRVQDZNl5W1oF5cfKfgIMlA==",
       "dev": true,
       "requires": {
         "bluebird": "^3.3.0",
         "body-parser": "^1.16.1",
-        "chokidar": "^2.0.3",
+        "braces": "^3.0.2",
+        "chokidar": "^3.0.0",
         "colors": "^1.1.0",
-        "combine-lists": "^1.0.0",
         "connect": "^3.6.0",
-        "core-js": "^2.2.0",
+        "core-js": "^3.1.3",
         "di": "^0.0.1",
         "dom-serialize": "^2.2.0",
-        "expand-braces": "^0.1.1",
         "flatted": "^2.0.0",
         "glob": "^7.1.1",
         "graceful-fs": "^4.1.2",
         "http-proxy": "^1.13.0",
         "isbinaryfile": "^3.0.0",
-        "lodash": "^4.17.5",
-        "log4js": "^3.0.0",
+        "lodash": "^4.17.11",
+        "log4js": "^4.0.0",
         "mime": "^2.3.1",
         "minimatch": "^3.0.2",
         "optimist": "^0.6.1",
@@ -6110,17 +6051,137 @@
         "useragent": "2.3.0"
       },
       "dependencies": {
-        "mime": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz",
-          "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==",
+        "anymatch": {
+          "version": "3.0.3",
+          "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.0.3.tgz",
+          "integrity": "sha512-c6IvoeBECQlMVuYUjSwimnhmztImpErfxJzWZhIQinIvQWoGOnB0dLIgifbPHQt5heS6mNlaZG16f06H3C8t1g==",
+          "dev": true,
+          "requires": {
+            "normalize-path": "^3.0.0",
+            "picomatch": "^2.0.4"
+          }
+        },
+        "binary-extensions": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
+          "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
           "dev": true
         },
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+          "dev": true,
+          "requires": {
+            "fill-range": "^7.0.1"
+          }
+        },
+        "chokidar": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.0.2.tgz",
+          "integrity": "sha512-c4PR2egjNjI1um6bamCQ6bUNPDiyofNQruHvKgHQ4gDUP/ITSVSzNsiI5OWtHOsX323i5ha/kk4YmOZ1Ktg7KA==",
+          "dev": true,
+          "requires": {
+            "anymatch": "^3.0.1",
+            "braces": "^3.0.2",
+            "fsevents": "^2.0.6",
+            "glob-parent": "^5.0.0",
+            "is-binary-path": "^2.1.0",
+            "is-glob": "^4.0.1",
+            "normalize-path": "^3.0.0",
+            "readdirp": "^3.1.1"
+          }
+        },
+        "core-js": {
+          "version": "3.1.4",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.1.4.tgz",
+          "integrity": "sha512-YNZN8lt82XIMLnLirj9MhKDFZHalwzzrL9YLt6eb0T5D0EDl4IQ90IGkua8mHbnxNrkj1d8hbdizMc0Qmg1WnQ==",
+          "dev": true
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "fsevents": {
+          "version": "2.0.7",
+          "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.0.7.tgz",
+          "integrity": "sha512-a7YT0SV3RB+DjYcppwVDLtn13UQnmg0SWZS7ezZD0UjnLwXmy8Zm21GMVGLaFGimIqcvyMQaOJBrop8MyOp1kQ==",
+          "dev": true,
+          "optional": true
+        },
+        "glob-parent": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.0.0.tgz",
+          "integrity": "sha512-Z2RwiujPRGluePM6j699ktJYxmPpJKCfpGA13jz2hmFZC7gKetzrWvg5KN3+OsIFmydGyZ1AVwERCq1w/ZZwRg==",
+          "dev": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        },
+        "is-binary-path": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+          "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+          "dev": true,
+          "requires": {
+            "binary-extensions": "^2.0.0"
+          }
+        },
+        "is-glob": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
+          "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
+          "dev": true,
+          "requires": {
+            "is-extglob": "^2.1.1"
+          }
+        },
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+          "dev": true
+        },
+        "mime": {
+          "version": "2.4.4",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
+          "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==",
+          "dev": true
+        },
+        "normalize-path": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
+          "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+          "dev": true
+        },
+        "readdirp": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.1.1.tgz",
+          "integrity": "sha512-XXdSXZrQuvqoETj50+JAitxz1UPdt5dupjT6T5nVB+WvjMv2XKYj+s7hPeAVCXvmJrL36O4YYyWlIC3an2ePiQ==",
+          "dev": true,
+          "requires": {
+            "picomatch": "^2.0.4"
+          }
+        },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
           "dev": true
+        },
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
         }
       }
     },
@@ -6294,9 +6355,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
+      "version": "4.17.15",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+      "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
       "dev": true
     },
     "lodash.clonedeep": {
@@ -6318,31 +6379,31 @@
       "dev": true
     },
     "log4js": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/log4js/-/log4js-3.0.6.tgz",
-      "integrity": "sha512-ezXZk6oPJCWL483zj64pNkMuY/NcRX5MPiB0zE6tjZM137aeusrOnW1ecxgF9cmwMWkBMhjteQxBPoZBh9FDxQ==",
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/log4js/-/log4js-4.5.1.tgz",
+      "integrity": "sha512-EEEgFcE9bLgaYUKuozyFfytQM2wDHtXn4tAN41pkaxpNjAykv11GVdeI4tHtmPWW4Xrgh9R/2d7XYghDVjbKKw==",
       "dev": true,
       "requires": {
-        "circular-json": "^0.5.5",
-        "date-format": "^1.2.0",
-        "debug": "^3.1.0",
-        "rfdc": "^1.1.2",
-        "streamroller": "0.7.0"
+        "date-format": "^2.0.0",
+        "debug": "^4.1.1",
+        "flatted": "^2.0.0",
+        "rfdc": "^1.1.4",
+        "streamroller": "^1.0.6"
       },
       "dependencies": {
         "debug": {
-          "version": "3.2.6",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
-          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
           "dev": true,
           "requires": {
             "ms": "^2.1.1"
           }
         },
         "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
           "dev": true
         }
       }
@@ -6753,9 +6814,9 @@
       }
     },
     "mixin-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
-      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
+      "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
       "dev": true,
       "requires": {
         "for-in": "^1.0.2",
@@ -7782,6 +7843,12 @@
       "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
       "dev": true
     },
+    "picomatch": {
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.0.7.tgz",
+      "integrity": "sha512-oLHIdio3tZ0qH76NybpeneBhYVj0QFTfXEFTc/B3zKQspYfYYkWYgFsmzo+4kvId/bQRcNkVeguI3y+CD22BtA==",
+      "dev": true
+    },
     "pify": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
@@ -8264,25 +8331,47 @@
       "dev": true
     },
     "raw-body": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz",
-      "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==",
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
+      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
       "dev": true,
       "requires": {
-        "bytes": "3.0.0",
-        "http-errors": "1.6.3",
-        "iconv-lite": "0.4.23",
+        "bytes": "3.1.0",
+        "http-errors": "1.7.2",
+        "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
       },
       "dependencies": {
-        "iconv-lite": {
-          "version": "0.4.23",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
-          "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==",
+        "bytes": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
+          "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==",
+          "dev": true
+        },
+        "http-errors": {
+          "version": "1.7.2",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
+          "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
           "dev": true,
           "requires": {
-            "safer-buffer": ">= 2.1.2 < 3"
+            "depd": "~1.1.2",
+            "inherits": "2.0.3",
+            "setprototypeof": "1.1.1",
+            "statuses": ">= 1.5.0 < 2",
+            "toidentifier": "1.0.0"
           }
+        },
+        "setprototypeof": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
+          "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==",
+          "dev": true
+        },
+        "statuses": {
+          "version": "1.5.0",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+          "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=",
+          "dev": true
         }
       }
     },
@@ -8633,9 +8722,9 @@
       "dev": true
     },
     "rfdc": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.1.2.tgz",
-      "integrity": "sha512-92ktAgvZhBzYTIK0Mja9uen5q5J3NRVMoDkJL2VMwq6SXjVCgqvQeVP2XAaUY6HT+XpQYeLSjb3UoitBryKmdA==",
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.1.4.tgz",
+      "integrity": "sha512-5C9HXdzK8EAqN7JDif30jqsBzavB7wLpaubisuQIGHWf2gUXSpzy6ArX/+Da8RjFpagWsCn+pIgxTMAmKw9Zug==",
       "dev": true
     },
     "rimraf": {
@@ -8950,9 +9039,9 @@
       "dev": true
     },
     "set-value": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
-      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
+      "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
       "dev": true,
       "requires": {
         "extend-shallow": "^2.0.1",
@@ -9652,15 +9741,16 @@
       "dev": true
     },
     "streamroller": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-0.7.0.tgz",
-      "integrity": "sha512-WREzfy0r0zUqp3lGO096wRuUp7ho1X6uo/7DJfTlEi0Iv/4gT7YHqXDjKC2ioVGBZtE8QzsQD9nx1nIuoZ57jQ==",
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-1.0.6.tgz",
+      "integrity": "sha512-3QC47Mhv3/aZNFpDDVO44qQb9gwB9QggMEE0sQmkTAwBVYdBRWISdsywlkfm5II1Q5y/pmrHflti/IgmIzdDBg==",
       "dev": true,
       "requires": {
-        "date-format": "^1.2.0",
-        "debug": "^3.1.0",
-        "mkdirp": "^0.5.1",
-        "readable-stream": "^2.3.0"
+        "async": "^2.6.2",
+        "date-format": "^2.0.0",
+        "debug": "^3.2.6",
+        "fs-extra": "^7.0.1",
+        "lodash": "^4.17.14"
       },
       "dependencies": {
         "debug": {
@@ -9672,10 +9762,21 @@
             "ms": "^2.1.1"
           }
         },
+        "fs-extra": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
+          "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.2",
+            "jsonfile": "^4.0.0",
+            "universalify": "^0.1.0"
+          }
+        },
         "ms": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
           "dev": true
         }
       }
@@ -10311,13 +10412,30 @@
       "dev": true
     },
     "type-is": {
-      "version": "1.6.16",
-      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
-      "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==",
+      "version": "1.6.18",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
+      "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
       "dev": true,
       "requires": {
         "media-typer": "0.3.0",
-        "mime-types": "~2.1.18"
+        "mime-types": "~2.1.24"
+      },
+      "dependencies": {
+        "mime-db": {
+          "version": "1.40.0",
+          "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
+          "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==",
+          "dev": true
+        },
+        "mime-types": {
+          "version": "2.1.24",
+          "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
+          "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
+          "dev": true,
+          "requires": {
+            "mime-db": "1.40.0"
+          }
+        }
       }
     },
     "typedarray": {
@@ -10359,38 +10477,15 @@
       "dev": true
     },
     "union-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
-      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
+      "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
       "dev": true,
       "requires": {
         "arr-union": "^3.1.0",
         "get-value": "^2.0.6",
         "is-extendable": "^0.1.1",
-        "set-value": "^0.4.3"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-          "dev": true,
-          "requires": {
-            "is-extendable": "^0.1.0"
-          }
-        },
-        "set-value": {
-          "version": "0.4.3",
-          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
-          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
-          "dev": true,
-          "requires": {
-            "extend-shallow": "^2.0.1",
-            "is-extendable": "^0.1.1",
-            "is-plain-object": "^2.0.1",
-            "to-object-path": "^0.3.0"
-          }
-        }
+        "set-value": "^2.0.1"
       }
     },
     "unique-filename": {

vripper-ui/package.json

@@ -1,12 +1,12 @@
 {
   "name": "vripper-ui",
-  "version": "1.4.3",
+  "version": "2.3.1",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
     "build": "ng build",
-    "build-electron": "ng build --base-href ./",
-    "prod": "ng build --prod",
+    "build-electron": "ng build --prod --base-href ./",
+    "build-web": "ng build --prod",
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e"
@@ -24,8 +24,8 @@
     "@angular/platform-browser": "~7.2.0",
     "@angular/platform-browser-dynamic": "~7.2.0",
     "@angular/router": "~7.2.0",
-    "ag-grid-angular": "^21.0.1",
-    "ag-grid-community": "^21.0.1",
+    "ag-grid-angular": "^21.2.1",
+    "ag-grid-community": "^21.2.1",
     "core-js": "^2.5.4",
     "hammerjs": "^2.0.8",
     "ngx-electron": "^2.1.1",
@@ -42,10 +42,10 @@
     "@types/jasminewd2": "~2.0.3",
     "@types/node": "~8.9.4",
     "codelyzer": "~4.5.0",
-    "electron": "^5.0.5",
+    "electron": "^6.0.9",
     "jasmine-core": "~2.99.1",
     "jasmine-spec-reporter": "~4.2.1",
-    "karma": "~3.1.1",
+    "karma": "^4.2.0",
     "karma-chrome-launcher": "~2.2.0",
     "karma-coverage-istanbul-reporter": "~2.0.1",
     "karma-jasmine": "~1.1.2",

vripper-ui/pom.xml

@@ -5,46 +5,111 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.3</version>
+    <version>2.3.1</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>
   <build>
     <plugins>
       <plugin>
-        <groupId>com.github.eirslett</groupId>
-        <artifactId>frontend-maven-plugin</artifactId>
-        <version>1.3</version>
+        <artifactId>maven-jar-plugin</artifactId>
         <configuration>
-          <nodeVersion>v12.2.0</nodeVersion>
-          <npmVersion>6.9.0</npmVersion>
-          <workingDirectory>build-dir</workingDirectory>
+          <classifier>${buildClassifier}</classifier>
         </configuration>
-        <executions>
-          <execution>
-            <id>install node and npm</id>
-            <goals>
-              <goal>install-node-and-npm</goal>
-            </goals>
-          </execution>
-          <execution>
-            <id>npm install</id>
-            <goals>
-              <goal>npm</goal>
-            </goals>
-          </execution>
-          <execution>
-            <id>npm run build</id>
-            <goals>
-              <goal>npm</goal>
-            </goals>
-            <configuration>
-              <arguments>run prod</arguments>
-            </configuration>
-            <phase>generate-resources</phase>
-          </execution>
-        </executions>
       </plugin>
     </plugins>
   </build>
+  <profiles>
+    <profile>
+      <id>web</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <properties>
+        <buildClassifier>web</buildClassifier>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.github.eirslett</groupId>
+            <artifactId>frontend-maven-plugin</artifactId>
+            <version>1.3</version>
+            <configuration>
+              <nodeVersion>v12.2.0</nodeVersion>
+              <npmVersion>6.9.0</npmVersion>
+              <workingDirectory>build-dir</workingDirectory>
+            </configuration>
+            <executions>
+              <execution>
+                <id>install node and npm</id>
+                <goals>
+                  <goal>install-node-and-npm</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>npm install</id>
+                <goals>
+                  <goal>npm</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>npm run build</id>
+                <goals>
+                  <goal>npm</goal>
+                </goals>
+                <configuration>
+                  <arguments>run build-web</arguments>
+                </configuration>
+                <phase>generate-resources</phase>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>electron</id>
+      <properties>
+        <buildClassifier>electron</buildClassifier>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.github.eirslett</groupId>
+            <artifactId>frontend-maven-plugin</artifactId>
+            <version>1.3</version>
+            <configuration>
+              <nodeVersion>v12.2.0</nodeVersion>
+              <npmVersion>6.9.0</npmVersion>
+              <workingDirectory>build-dir</workingDirectory>
+            </configuration>
+            <executions>
+              <execution>
+                <id>install node and npm</id>
+                <goals>
+                  <goal>install-node-and-npm</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>npm install</id>
+                <goals>
+                  <goal>npm</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>npm run build</id>
+                <goals>
+                  <goal>npm</goal>
+                </goals>
+                <configuration>
+                  <arguments>run build-electron</arguments>
+                </configuration>
+                <phase>generate-resources</phase>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>

vripper-ui/src/app/app.component.html

@@ -1,33 +1,17 @@
 <div fxLayout="column" *ngIf="connecting()" class="overlay loading" fxLayoutAlign="center center">
   <mat-spinner></mat-spinner>
-  <h2>Connecting</h2>
+  <h2>Loading...</h2>
 </div>
 <div fxLayout="column" *ngIf="noConnectionState()" class="overlay no-connection" fxLayoutAlign="center center">
   <mat-icon class="overlay-icon">error</mat-icon>
-  <h2>Connection with the server is closed</h2>
+  <h2>The app seems to be shutdown</h2>
 </div>
-<div [ngClass]="{'electron': electronService.isElectronApp}" id="app-container" fxLayout="column" fxLayoutGap="20px">
+<div [ngClass]="{ electron: electronService.isElectronApp }" fxLayout="column" id="app-container">
   <div id="app-header" fxFlex="nogrow">
-    <mat-toolbar class="mat-elevation-z2" color="primary">
-      <mat-toolbar-row fxLayout="row" fxLayoutAlign="center center">
-        <span id="logo" fxFlex="0 0 48px"></span>
-        <span id="title">Viper Ripper</span>
-        <span fxFlex="grow"></span>
-        <div>
-          <button mat-icon-button [matMenuTriggerFor]="menu" aria-label="settings">
-            <mat-icon>more_vert</mat-icon>
-          </button>
-          <mat-menu #menu="matMenu">
-            <button (click)="openSettings()" mat-menu-item>
-              <mat-icon>settings</mat-icon>
-              <span>Settings</span>
-            </button>
-          </mat-menu>
-        </div>
-      </mat-toolbar-row>
-    </mat-toolbar>
+    <app-toolbar></app-toolbar>
   </div>
   <div id="app-body" fxFlex="grow">
     <router-outlet></router-outlet>
+    <app-status-bar style="width: 100%"></app-status-bar>
   </div>
 </div>

vripper-ui/src/app/app.component.scss

@@ -1,11 +1,3 @@
-#logo {
-  height: 48px;
-  margin-right: 10px;
-  background-image: url('../assets/logo.png');
-  background-repeat: no-repeat;
-  background-size: 48px 48px;
-}
-
 #title {
   user-select: none
 }
@@ -19,7 +11,3 @@
 .no-connection {
   color: #eb6060;
 }
-
-.electron {
-  margin-top: 30px;
-}

vripper-ui/src/app/app.component.scss-theme.scss

@@ -0,0 +1,11 @@
+@import '~@angular/material/theming';
+
+@mixin app-component-theme($theme) {
+
+  $background: map-get($theme, background);
+
+  button.add-button {
+
+    background-color: mat-color($background, background);
+  }
+}

vripper-ui/src/app/app.component.ts

@@ -1,69 +1,63 @@
+import { AppService } from './app.service';
 import { ClipboardService } from './clipboard.service';
 import { ElectronService } from 'ngx-electron';
-import { SettingsComponent } from './settings/settings.component';
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, OnDestroy, NgZone, AfterViewInit, Renderer2 } from '@angular/core';
 import { MatDialog } from '@angular/material';
-import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout';
-import { Observable } from 'rxjs';
+import { Subscription } from 'rxjs';
 import { WsConnectionService, WSState } from './ws-connection.service';
+import { WsHandler } from './ws-handler';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
   styleUrls: ['./app.component.scss']
 })
-export class AppComponent implements OnInit {
+export class AppComponent implements OnInit, OnDestroy, AfterViewInit {
   constructor(
     public dialog: MatDialog,
-    private breakpointObserver: BreakpointObserver,
     private ws: WsConnectionService,
     public electronService: ElectronService,
-    private clipboardService: ClipboardService
-    ) {
-      this.currentState = WSState.INIT;
-      this.ws.state.subscribe(e => {
-        this.currentState = e;
-        if (this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR) {
-          this.dialog.closeAll();
-        } else if(this.currentState === WSState.OPEN) {
-          this.clipboardService.init();
-        }
-      });
-    }
+    private clipboardService: ClipboardService,
+    private ngZone: NgZone,
+    private appService: AppService,
+    private renderer: Renderer2
+  ) {}
 
+  subscriptions: Subscription[] = [];
+  websocketHandlerPromise: Promise<WsHandler>;
   currentState: WSState;
-
-  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
-
-  openSettings(): void {
-    const dialogRef = this.dialog.open(SettingsComponent, {
-      width: '70%',
-      height: '70%',
-      maxWidth: '100vw',
-      maxHeight: '100vh'
-    });
-
-    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
-      if (result.matches) {
-        dialogRef.updateSize('100%', '100%');
-      } else {
-        dialogRef.updateSize('70%', '70%');
-      }
-    });
-
-    dialogRef.afterClosed().subscribe(result => {
-      smallDialogSubscription.unsubscribe();
-    });
-  }
+  themeLoaded = false;
 
   noConnectionState(): boolean {
     return this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR;
   }
 
   connecting(): boolean {
-    return this.currentState === WSState.INIT || this.currentState === WSState.CONNECTING;
+    return (this.currentState === WSState.INIT || this.currentState === WSState.CONNECTING) && !this.themeLoaded ;
+  }
+
+  ngAfterViewInit() {
+    this.appService.renderer = this.renderer;
   }
 
   ngOnInit() {
+    this.currentState = WSState.INIT;
+    this.subscriptions.push(this.ws.state.subscribe(e => {
+      this.currentState = e;
+      if (this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR) {
+        this.dialog.closeAll();
+      } else if (this.currentState === WSState.OPEN) {
+        this.clipboardService.init();
+        this.appService
+          .loadTheme()
+          .subscribe(() => this.ngZone.run(() => this.themeLoaded = true));
+      }
+    }));
+  }
+
+  ngOnDestroy() {
+    this.subscriptions.forEach(e => {
+      e.unsubscribe();
+    });
   }
 }

vripper-ui/src/app/app.module.ts

@@ -1,3 +1,5 @@
+import { PostsDataService } from './posts-data.service';
+import { AppPreviewDirective } from './common/preview-tooltip.directive';
 import { WsConnectionService } from './ws-connection.service';
 import { AppService } from './app.service';
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
@@ -13,7 +15,6 @@ import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { AgGridModule } from 'ag-grid-angular';
 import { PostProgressRendererComponent } from './posts/post-progress.renderer.component';
-import { MenuRendererComponent } from './posts/menu.renderer.component';
 import { PostDetailsProgressRendererComponent } from './post-detail/post-details-progress.component';
 import { LoginComponent } from './login/login.component';
 import { XhrInterceptorService } from './xhr-interceptor.service';
@@ -23,6 +24,16 @@ import { SettingsComponent } from './settings/settings.component';
 import { NgxElectronModule } from 'ngx-electron';
 import { ServerService } from './server-service';
 import { ConfirmDialogComponent } from './common/confirmation-component/confirmation-dialog';
+import { SharedService } from './posts/shared.service';
+import { MultiPostComponent } from './multi-post/multi-post.component';
+import { OverlayModule } from '@angular/cdk/overlay';
+import { AppPreviewComponent } from './common/preview-tooltip.component';
+import { UrlRendererComponent } from './multi-post/url-renderer.component';
+import { FilterComponent } from './filter/filter.component';
+import { ScanComponent } from './scan/scan.component';
+import { StatusBarComponent } from './status-bar/status-bar.component';
+import { SelectionService } from './selection-service';
+import { ToolbarComponent } from './toolbar/toolbar.component';
 
 @NgModule({
   declarations: [
@@ -30,17 +41,26 @@ import { ConfirmDialogComponent } from './common/confirmation-component/confirma
     PostsComponent,
     PostDetailComponent,
     PostProgressRendererComponent,
-    MenuRendererComponent,
     PostDetailsProgressRendererComponent,
     LoginComponent,
     HomeComponent,
     SettingsComponent,
-    ConfirmDialogComponent
+    ConfirmDialogComponent,
+    MultiPostComponent,
+    AppPreviewComponent,
+    AppPreviewDirective,
+    UrlRendererComponent,
+    FilterComponent,
+    ScanComponent,
+    StatusBarComponent,
+    ToolbarComponent
   ],
   entryComponents: [
     PostDetailComponent,
     SettingsComponent,
-    ConfirmDialogComponent
+    ConfirmDialogComponent,
+    AppPreviewComponent,
+    ScanComponent
   ],
   imports: [
     BrowserAnimationsModule,
@@ -51,9 +71,22 @@ import { ConfirmDialogComponent } from './common/confirmation-component/confirma
     AppRoutingModule,
     NgxElectronModule,
     ReactiveFormsModule,
-    AgGridModule.withComponents([PostProgressRendererComponent, MenuRendererComponent, PostDetailsProgressRendererComponent])
+    AgGridModule.withComponents([
+      PostProgressRendererComponent,
+      PostDetailsProgressRendererComponent,
+      UrlRendererComponent
+    ]),
+    OverlayModule
+  ],
+  providers: [
+    AppService,
+    WsConnectionService,
+    { provide: HTTP_INTERCEPTORS, useClass: XhrInterceptorService, multi: true },
+    ServerService,
+    SharedService,
+    SelectionService,
+    PostsDataService
   ],
-  providers: [AppService, WsConnectionService, { provide: HTTP_INTERCEPTORS, useClass: XhrInterceptorService, multi: true }, ServerService],
   bootstrap: [AppComponent]
 })
-export class AppModule { }
+export class AppModule {}

vripper-ui/src/app/app.service.ts

@@ -1,8 +1,79 @@
-import { Injectable } from '@angular/core';
+import { MatDialog } from '@angular/material';
+import { HttpClient } from '@angular/common/http';
+import { Injectable, Renderer2, NgZone } from '@angular/core';
+import { ServerService } from './server-service';
+import { tap } from 'rxjs/operators';
+import { ScanComponent } from './scan/scan.component';
+import { Observable } from 'rxjs';
+import { BreakpointState, Breakpoints, BreakpointObserver } from '@angular/cdk/layout';
 
 @Injectable()
 export class AppService {
+  constructor(
+    private httpClient: HttpClient,
+    private serverService: ServerService,
+    public dialog: MatDialog,
+    private breakpointObserver: BreakpointObserver,
+  ) {}
 
-  constructor() {
+  darkTheme = false;
+  _renderer: Renderer2;
+  set renderer(renderer: Renderer2) {
+    this._renderer = renderer;
+  }
+
+  isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
+  isScanOpen = false;
+
+  updateTheme(darkTheme: boolean) {
+    this.darkTheme = darkTheme;
+    if (this.darkTheme) {
+      this._renderer.addClass(document.body, 'dark-theme');
+      this._renderer.removeClass(document.body, 'light-theme');
+    } else {
+      this._renderer.addClass(document.body, 'light-theme');
+      this._renderer.removeClass(document.body, 'dark-theme');
+    }
+    this.httpClient
+      .post<Theme>(this.serverService.baseUrl + '/settings/theme', {
+        darkTheme: this.darkTheme
+      })
+      .subscribe();
+  }
+
+  loadTheme() {
+    return this.httpClient
+      .get<Theme>(this.serverService.baseUrl + '/settings/theme')
+      .pipe(tap(theme => this.updateTheme(theme.darkTheme)));
+  }
+
+  scan(url?: string) {
+
+    const scanDialog = this.dialog.open(ScanComponent, {
+      width: '70%',
+      height: '70%',
+      maxWidth: '100vw',
+      maxHeight: '100vh',
+      data: {url: url}
+    });
+
+    const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
+      if (result.matches) {
+        scanDialog.updateSize('100%', '100%');
+      } else {
+        scanDialog.updateSize('70%', '70%');
+      }
+    });
+
+    scanDialog.afterOpened().subscribe(() => this.isScanOpen = true);
+
+    scanDialog.afterClosed().subscribe(() => {
+      smallDialogSubscription.unsubscribe();
+      this.isScanOpen = false;
+    });
   }
 }
+
+export interface Theme {
+  darkTheme: boolean;
+}

vripper-ui/src/app/clipboard.service.ts

@@ -12,6 +12,7 @@ import { ServerService } from './server-service';
 export class ClipboardService {
   private links$: Subject<string> = new Subject();
   private interval: NodeJS.Timer;
+  private lastText = '';
   constructor(
     private electronService: ElectronService,
     private serverService: ServerService,
@@ -47,12 +48,11 @@ export class ClipboardService {
     }
 
     const clipboard: Clipboard = this.electronService.clipboard;
-    let lastText = '';
     this.interval = setInterval(() => {
       const text = clipboard.readText();
 
-      if (this.textHasDiff(text, lastText)) {
-        lastText = text;
+      if (this.textHasDiff(text, this.lastText)) {
+        this.lastText = text;
         if (text.indexOf('https://vipergirls.to/threads') !== -1) {
           this.links$.next(text);
         }

vripper-ui/src/app/common/cmd.enum.ts

@@ -1,6 +1,14 @@
 export enum CMD {
     POSTS_SUB = 'POSTS_SUB',
     POST_DETAILS_SUB = 'POST_DETAILS_SUB',
+    GLOBAL_STATE_SUB = 'GLOBAL_STATE_SUB',
+    SPEED_SUB = 'SPEED_SUB',
+    USER_SUB = 'USER_SUB',
+    THREAD_PARSING_SUB = 'THREAD_PARSING_SUB',
     POSTS_UNSUB = 'POSTS_UNSUB',
-    POST_DETAILS_UNSUB = 'POST_DETAILS_UNSUB'
+    POST_DETAILS_UNSUB = 'POST_DETAILS_UNSUB',
+    GLOBAL_STATE_UNSUB = 'GLOBAL_STATE_UNSUB',
+    SPEED_UNSUB = 'SPEED_UNSUB',
+    USER_UNSUB = 'USER_UNSUB',
+    THREAD_PARSING_UNSUB = 'THREAD_PARSING_UNSUB'
 }

vripper-ui/src/app/common/download-path.model.ts

@@ -0,0 +1,3 @@
+export interface DownloadPath {
+    path: string;
+}

vripper-ui/src/app/common/download-speed.model.ts

@@ -0,0 +1,3 @@
+export class DownloadSpeed {
+    constructor(public speed: string) {}
+}

vripper-ui/src/app/common/global-state.model.ts

@@ -0,0 +1,3 @@
+export class GlobalState {
+    constructor(public running: number, public queued: number,  public remaining: number,  public error: number) {}
+}

vripper-ui/src/app/common/logged-user.model.ts

@@ -0,0 +1,3 @@
+export class LoggedUser {
+    constructor(public user: string) {}
+}

vripper-ui/src/app/common/parse-response.model.ts

@@ -1,3 +0,0 @@
-export interface ParseResponse {
-    parsed: number;
-}

vripper-ui/src/app/common/preview-tooltip.component.ts

@@ -0,0 +1,36 @@
+import { Component, Input } from '@angular/core';
+import { trigger, state, style, transition, animate } from '@angular/animations';
+
+@Component({
+  selector: 'app-preview',
+  template: `
+  <div class="previews" style="display: inline-block; white-space: nowrap;">
+    <ng-container *ngFor="let link of links">
+      <img style="display: inline-block; max-width:150px; min-width:150px; width: auto; height: auto;"
+        [@simpleFadeAnimation]="'in'"
+        class="mat-elevation-z8"
+        [src]="link"
+      />
+    </ng-container>
+  </div>
+  `,
+  styles: [
+    `
+      img {
+        max-height: 200px;
+        max-width: 200px;
+        margin-left: 5px;
+      }
+    `
+  ],
+  animations: [
+    trigger('simpleFadeAnimation', [
+      state('in', style({ opacity: 1 })),
+
+      transition(':enter', [style({ opacity: 0 }), animate(300)])
+    ])
+  ]
+})
+export class AppPreviewComponent {
+  @Input() links: string[] = [];
+}

vripper-ui/src/app/common/preview-tooltip.directive.ts

@@ -0,0 +1,46 @@
+import { AppPreviewComponent } from './preview-tooltip.component';
+import { Directive, Input, HostListener, ComponentRef, OnInit, ElementRef, NgZone } from '@angular/core';
+import { OverlayRef, Overlay, OverlayPositionBuilder } from '@angular/cdk/overlay';
+import { ComponentPortal } from '@angular/cdk/portal';
+
+@Directive({ selector: '[appPreview]' })
+export class AppPreviewDirective implements OnInit {
+  private overlayRef: OverlayRef;
+  tooltipPortal = new ComponentPortal(AppPreviewComponent);
+
+  @Input() appPreview: string[];
+
+  constructor(private overlayPositionBuilder: OverlayPositionBuilder,
+    private elementRef: ElementRef,
+    private overlay: Overlay,
+    private ngZone: NgZone) {}
+
+  ngOnInit() {
+    const positionStrategy = this.overlayPositionBuilder
+    .flexibleConnectedTo(this.elementRef)
+    .withPush(true)
+    .withGrowAfterOpen(true)
+    .withPositions([{
+      originX: 'start',
+      originY: 'bottom',
+      overlayX: 'start',
+      overlayY: 'top',
+    }]);
+    this.overlayRef = this.overlay.create({ positionStrategy });
+  }
+
+  @HostListener('mouseenter')
+  show() {
+    this.ngZone.run(() => {
+      const tooltipRef: ComponentRef<AppPreviewComponent> = this.overlayRef.attach(this.tooltipPortal);
+      tooltipRef.instance.links = this.appPreview;
+    });
+  }
+
+  @HostListener('mouseout')
+  hide() {
+    this.ngZone.run(() => {
+      this.overlayRef.detach();
+    });
+  }
+}

vripper-ui/src/app/common/vr-post-parse.model.ts

@@ -0,0 +1,15 @@
+export class VRPostParse {
+  constructor(
+    public threadId: string,
+    public postId: string,
+    public number: number,
+    public title: string,
+    public imageCount: number,
+    public url: string,
+    public previews: string[]
+  ) {}
+}
+
+export class VRThreadParseState {
+  constructor(public threadId: string, public state: string) {}
+}

vripper-ui/src/app/filter/filter.component.html

@@ -0,0 +1,3 @@
+<mat-chip-list>
+  <mat-chip (click)="select(c)" *ngFor="let c of criteria" [selected]="c.selected" color="accent">{{c.value}}</mat-chip>
+</mat-chip-list>

vripper-ui/src/app/filter/filter.component.ts

@@ -0,0 +1,26 @@
+import { Component, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'app-filter',
+  templateUrl: './filter.component.html',
+  styleUrls: ['./filter.component.scss']
+})
+export class FilterComponent implements OnInit {
+
+  constructor() { }
+
+  criteria = [
+    {value: 'All', selected: true},
+    {value: 'Downloading', selected: false},
+    {value: 'Completed', selected: false},
+    {value: 'Error', selected: false},
+  ];
+
+  select(chip: {value: string, selected: boolean}) {
+    this.criteria.forEach(e => e.selected = false);
+    chip.selected = true;
+  }
+
+  ngOnInit() {
+  }
+}

vripper-ui/src/app/home/home.component.html

@@ -1,36 +1,3 @@
-<div *ngIf="loading" class="overlay loading" fxLayout="row" fxLayoutAlign="center center">
-  <mat-spinner></mat-spinner>
-</div>
-<div fxLayout="column" fxLayoutGap="10px" style="height: 100%">
-  <div fxFlex="nogrow">
-    <form
-      #f="ngForm"
-      (ngSubmit)="submit(f)"
-      autocomplete="off"
-      fxLayout="row"
-      fxLayoutAlign="center center"
-      fxLayoutGap="20px"
-    >
-      <mat-form-field fxFlex="grow">
-        <input [(ngModel)]="input" matInput name="url" placeholder="Put a vipergirls.to link" required/>
-      </mat-form-field>
-      <div>
-        <button type="submit" mat-raised-button color="primary" [disabled]="f.invalid">Grab</button>
-      </div>
-    </form>
-  </div>
-  <div fxLayout="row" fxLayoutAlign="end" fxLayoutGap="10px">
-    <button (click)="clear()" aria-label="Clear completed" color="primary" mat-stroked-button title="Clear completed">
-      Clear completed
-      <mat-icon>clear_all</mat-icon>
-    </button>
-    <button (click)="remove()" aria-label="Remove All" color="primary" mat-stroked-button title="Remove All">
-      Remove All
-      <mat-icon>delete_forever</mat-icon>
-    </button>
-  </div>
-  <div fxLayout="column" fxFlex="grow">
-    <app-posts [ngStyle]="{'height': electronService.isElectronApp ? 'calc(100% - 27px)' : '100%'}" fxFlex="nogrow"
-               style="width: 100%; overflow-x: hidden"></app-posts>
-  </div>
+<div style="height: calc(100% - 20px)">
+  <app-posts style="width: 100%; height: 100%;"></app-posts>
 </div>

vripper-ui/src/app/home/home.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { HomeComponent } from './home.component';
-
-describe('HomeComponent', () => {
-  let component: HomeComponent;
-  let fixture: ComponentFixture<HomeComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ HomeComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(HomeComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

vripper-ui/src/app/home/home.component.ts

@@ -1,122 +1,40 @@
+import { AppService } from './../app.service';
 import { ElectronService } from 'ngx-electron';
 import { ClipboardService } from './../clipboard.service';
-import { ParseResponse } from './../common/parse-response.model';
-import { Component, OnInit, ViewChild, Inject } from '@angular/core';
-import { HttpClient } from '@angular/common/http';
-import { finalize, filter, flatMap } from 'rxjs/operators';
-import { MatSnackBar, MatDialogRef, MAT_DIALOG_DATA, MatDialog } from '@angular/material';
-import { NgForm } from '@angular/forms';
-import { ServerService } from '../server-service';
-import { RemoveAllResponse } from '../common/remove-all-response.model';
-import { PostsComponent } from '../posts/posts.component';
-import { ConfirmDialogComponent } from '../common/confirmation-component/confirmation-dialog';
+import { Component, OnInit, OnDestroy, NgZone, ChangeDetectionStrategy } from '@angular/core';
+import { MatDialog, MatSnackBar } from '@angular/material';
 
 @Component({
   selector: 'app-home',
   templateUrl: './home.component.html',
-  styleUrls: ['./home.component.scss']
+  styleUrls: ['./home.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
-export class HomeComponent implements OnInit {
+export class HomeComponent implements OnInit, OnDestroy {
+
   constructor(
-    private httpClient: HttpClient,
-    private _snackBar: MatSnackBar,
-    private serverService: ServerService,
     private clipboardService: ClipboardService,
     public dialog: MatDialog,
-    public electronService: ElectronService
-  ) {
+    public electronService: ElectronService,
+    private appService: AppService,
+    private _snackBar: MatSnackBar,
+    private ngZone: NgZone
+  ) {}
+
+  ngOnInit() {
     this.clipboardService.links.subscribe(e => {
-      this.loading = true;
-      this.httpClient
-        .post<ParseResponse>(this.serverService.baseUrl + '/clipboard/post', { url: e })
-        .pipe(
-          finalize(() => {
-            this.loading = false;
-          })
-        )
-        .subscribe(
-          data => {
-            this._snackBar.open(`${data.parsed} posts parsed`, null, { duration: 5000 });
-          },
-          error => {
-            this._snackBar.open(error.error, null, {
-              duration: 5000
-            });
-          }
-        );
+      this.ngZone.run(() => {
+        if (this.appService.isScanOpen) {
+          this._snackBar.open(e + ' was not processed, the app is busy parsing another thread', null, {
+            duration: 5000
+          });
+          return;
+        }
+        this.appService.scan(e);
+      });
     });
   }
 
-  @ViewChild(PostsComponent)
-  private postsComponent: PostsComponent;
-
-  loading = false;
-  input: string;
-
-  ngOnInit() {}
-
-  submit(form: NgForm) {
-    this.loading = true;
-    this.httpClient
-      .post<ParseResponse>(this.serverService.baseUrl + '/post', { url: this.input })
-      .pipe(
-        finalize(() => {
-          this.loading = false;
-          this.input = null;
-          form.resetForm();
-        })
-      )
-      .subscribe(
-        data => {
-          this._snackBar.open(`${data.parsed} posts parsed`, null, { duration: 5000 });
-        },
-        error => {
-          this._snackBar.open(error.error, null, {
-            duration: 5000
-          });
-        }
-      );
-  }
-
-  clear() {
-    this.httpClient.post<RemoveAllResponse>(this.serverService.baseUrl + '/post/clear/all', {}).subscribe(
-      data => {
-        this.postsComponent.removeRows(data.postIds);
-        this._snackBar.open(`${data.removed} items cleared`, null, { duration: 5000 });
-      },
-      error => {
-        this._snackBar.open(error.error, null, {
-          duration: 5000
-        });
-      }
-    );
-  }
-
-  remove() {
-
-    this.dialog
-    .open(ConfirmDialogComponent, {
-      maxHeight: '100vh',
-      maxWidth: '100vw',
-      height: '200px',
-      width: '60%',
-      data: {header: 'Confirmation', content: 'Are you sure you want to remove all items ?'}
-    })
-    .afterClosed()
-    .pipe(
-      filter(e => e === 'yes'),
-      flatMap(e => this.httpClient.post<RemoveAllResponse>(this.serverService.baseUrl + '/post/remove/all', {}))
-    )
-    .subscribe(
-      data => {
-        this.postsComponent.removeRows(data.postIds);
-        this._snackBar.open(`${data.removed} items removed`, null, { duration: 5000 });
-      },
-      error => {
-        this._snackBar.open(error.error, null, {
-          duration: 5000
-        });
-      }
-    );
+  ngOnDestroy() {
   }
 }

vripper-ui/src/app/login/login.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { LoginComponent } from './login.component';
-
-describe('LoginComponent', () => {
-  let component: LoginComponent;
-  let fixture: ComponentFixture<LoginComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ LoginComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(LoginComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

vripper-ui/src/app/material.module.ts

@@ -14,7 +14,9 @@ import {
     MatSlideToggleModule,
     MatSnackBarModule,
     MatSnackBar,
-    MatTabsModule
+    MatTabsModule,
+    MatDividerModule,
+    MatChipsModule
 } from '@angular/material';
 
 @NgModule({
@@ -32,7 +34,9 @@ import {
         MatCardModule,
         MatSlideToggleModule,
         MatSnackBarModule,
-        MatTabsModule
+        MatTabsModule,
+        MatDividerModule,
+        MatChipsModule
     ],
     providers: [MatSnackBar]
 })

vripper-ui/src/app/multi-post/multi-post.component.html

@@ -0,0 +1,14 @@
+<div fxLayout="column" style="height: 100%;">
+  <div fxFlex="nogrow">
+    <form autocomplete="off" fxLayout="row" fxLayoutAlign="left center" fxLayoutGap="20px">
+      <mat-form-field fxFlex="1 1 50%">
+        <input (ngModelChange)="search($event)" matInput name="search" ngModel placeholder="Search"/>
+      </mat-form-field>
+    </form>
+  </div>
+  <ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-material" style="width: 100%; height: 100%;">
+  </ag-grid-angular>
+  <div style="height: 4px; width: 100%; padding: 5px 0 0">
+    <mat-progress-bar *ngIf="loading | async" mode="indeterminate"></mat-progress-bar>
+  </div>
+</div>

vripper-ui/src/app/multi-post/multi-post.component.scss

@@ -0,0 +1,9 @@
+$cell-space: 8px;
+
+.title-cell {
+    margin-right: $cell-space;
+}
+
+.progress-cell {
+    margin-left: $cell-space;
+}

vripper-ui/src/app/multi-post/multi-post.component.ts

@@ -0,0 +1,165 @@
+import { VRPostParse, VRThreadParseState } from './../common/vr-post-parse.model';
+import { Component, OnInit, NgZone, OnDestroy, Input, Output, EventEmitter, ChangeDetectionStrategy, AfterViewInit } from '@angular/core';
+import { MatSnackBar } from '@angular/material';
+import { GridOptions } from 'ag-grid-community';
+import { UrlRendererComponent } from './url-renderer.component';
+import { WsHandler } from '../ws-handler';
+import { Subscription } from 'rxjs';
+import { WSMessage } from '../common/ws-message.model';
+import { CMD } from '../common/cmd.enum';
+import { WsConnectionService } from '../ws-connection.service';
+import { HttpClient } from '@angular/common/http';
+import { ServerService } from '../server-service';
+
+@Component({
+  selector: 'app-multi-post',
+  templateUrl: './multi-post.component.html',
+  styleUrls: ['./multi-post.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
+
+  @Input()
+  threadId: string;
+
+  @Output()
+  done: EventEmitter<boolean> = new EventEmitter();
+
+  gridOptions: GridOptions;
+  websocketHandlerPromise: Promise<WsHandler>;
+  subscription: Subscription;
+  loading: EventEmitter<boolean> = new EventEmitter();
+
+  constructor(
+    private ngZone: NgZone,
+    private _snackBar: MatSnackBar,
+    private wsConnectionService: WsConnectionService,
+    private httpClient: HttpClient,
+    private serverService: ServerService
+  ) {
+    this.websocketHandlerPromise = this.wsConnectionService.getConnection();
+  }
+
+  ngAfterViewInit(): void {
+    this.loading.emit(true);
+  }
+
+  ngOnInit(): void {
+    this.gridOptions = <GridOptions>{
+      columnDefs: [
+        {
+          headerName: 'Number',
+          field: 'number',
+          cellClass: 'no-padding',
+          maxWidth: 100,
+          checkboxSelection: true,
+          headerCheckboxSelection: true,
+          headerCheckboxSelectionFilteredOnly: false,
+          sort: 'asc'
+        },
+        {
+          headerName: 'Title',
+          field: 'title',
+          cellClass: 'no-padding'
+        },
+        {
+          headerName: 'URL',
+          field: 'url',
+          cellClass: 'no-padding',
+          cellRenderer: 'urlCellRenderer'
+        }
+      ],
+      defaultColDef: {
+        sortable: true
+      },
+      frameworkComponents: {
+        urlCellRenderer: UrlRendererComponent
+      },
+      rowSelection: 'multiple',
+      rowMultiSelectWithClick: true,
+      suppressRowClickSelection: true,
+      rowHeight: 48,
+      animateRows: true,
+      rowData: [],
+      overlayLoadingTemplate: '<span></span>',
+      overlayNoRowsTemplate: '<span></span>',
+      getRowNodeId: data => data['url'],
+      onGridReady: () => {
+        this.gridOptions.api.sizeColumnsToFit();
+        this.connect();
+      },
+      onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
+      onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit()
+    };
+  }
+
+  ngOnDestroy() {
+    this.unsubscribe();
+  }
+
+  search(event) {
+    this.gridOptions.api.setQuickFilter(event);
+  }
+
+  private unsubscribe() {
+    if (this.subscription != null) {
+      this.subscription.unsubscribe();
+    }
+    this.websocketHandlerPromise.then((handler: WsHandler) => {
+      handler.send(new WSMessage(CMD.THREAD_PARSING_UNSUB.toString()));
+    });
+  }
+
+  connect() {
+    this.websocketHandlerPromise.then((handler: WsHandler) => {
+      console.log('Connecting to thread parsing');
+      this.subscription = handler.subscribeForThreadParsing(
+        (states: Array<VRThreadParseState>, data: Array<VRPostParse>) => {
+          if (data[0].threadId === this.threadId) {
+            this.gridOptions.api.updateRowData({ add: data });
+          }
+
+          if (states.length > 0 && states[states.length - 1].state === 'END' && states[0].threadId === this.threadId) {
+            this.ngZone.run(() => {
+              this.loading.emit(false);
+              if (this.gridOptions.api.getDisplayedRowCount() === 1) {
+                this.addPosts(
+                  [this.gridOptions.api.getDisplayedRowAtIndex(0).data].map(e => ({
+                    threadId: e.threadId,
+                    postId: e.postId
+                  }))
+                );
+                this.done.emit(true);
+              }
+            });
+          }
+        }
+      );
+      handler.send(new WSMessage(CMD.THREAD_PARSING_SUB.toString(), this.threadId));
+    });
+  }
+
+  submit() {
+    const data = (<VRPostParse[]>this.gridOptions.api.getSelectedRows()).map(e => ({
+      postId: e.postId,
+      threadId: this.threadId
+    }));
+    this.addPosts(data);
+  }
+
+  addPosts(data: { postId: string; threadId: string }[]) {
+    this.httpClient.post(this.serverService.baseUrl + '/post/add', data).subscribe(
+      () => {
+        this._snackBar.open('Adding posts to queue', null, {
+          duration: 5000
+        });
+        this.done.emit();
+      },
+      error => {
+        this._snackBar.open(error.error, null, {
+          duration: 5000
+        });
+      }
+    );
+  }
+}

vripper-ui/src/app/multi-post/url-renderer.component.ts

@@ -0,0 +1,42 @@
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy } from '@angular/core';
+import { AgRendererComponent } from 'ag-grid-angular';
+import { ICellRendererParams } from 'ag-grid-community';
+import { VRPostParse } from '../common/vr-post-parse.model';
+import { ElectronService } from 'ngx-electron';
+
+@Component({
+  selector: 'app-url-cell',
+  template: `
+    <a href="javascript:void(0)" [appPreview]="postResult.previews" (click)="goTo()"
+      >https://vipergirls/threads/?p={{ postResult.postId }}</a
+    >
+  `,
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class UrlRendererComponent implements OnInit, OnDestroy, AgRendererComponent {
+  constructor(public electronService: ElectronService) {}
+
+  postResult: VRPostParse;
+  params: ICellRendererParams;
+
+  ngOnInit(): void {}
+
+  ngOnDestroy(): void {}
+
+  agInit(params: ICellRendererParams): void {
+    this.params = params;
+    this.postResult = this.params.data;
+  }
+
+  refresh(params: ICellRendererParams): boolean {
+    return false;
+  }
+
+  goTo() {
+    if (this.electronService.isElectronApp) {
+      this.electronService.shell.openExternal(this.postResult.url);
+    } else {
+      window.open(this.postResult.url, '_blank');
+    }
+  }
+}

vripper-ui/src/app/post-detail/post-detail.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { PostDetailComponent } from './post-detail.component';
-
-describe('PostDetailComponent', () => {
-  let component: PostDetailComponent;
-  let fixture: ComponentFixture<PostDetailComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ PostDetailComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(PostDetailComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

vripper-ui/src/app/post-detail/post-detail.component.ts

@@ -1,7 +1,7 @@
 import { PostDetailsProgressRendererComponent } from './post-details-progress.component';
 import { WsConnectionService } from './../ws-connection.service';
-import { Component, OnInit, ViewChild, Inject, OnDestroy, NgZone } from '@angular/core';
-import { MatSort, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material';
+import { Component, OnInit, Inject, OnDestroy, NgZone, ChangeDetectionStrategy } from '@angular/core';
+import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material';
 import { PostDetailsDataSource } from './post-details.datasource';
 import { PostState } from '../posts/post-state.model';
 import { GridOptions } from 'ag-grid-community';
@@ -9,7 +9,8 @@ import { GridOptions } from 'ag-grid-community';
 @Component({
   selector: 'app-post-detail',
   templateUrl: './post-detail.component.html',
-  styleUrls: ['./post-detail.component.scss']
+  styleUrls: ['./post-detail.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
 export class PostDetailComponent implements OnInit, OnDestroy {
 
@@ -25,12 +26,12 @@ export class PostDetailComponent implements OnInit, OnDestroy {
           headerName: 'URL',
           field: 'url',
           sortable: true,
-          tooltipField: 'url',
           cellRenderer: 'progressCellRenderer',
           cellClass: 'no-padding'
         }
       ],
       rowHeight: 48,
+      animateRows: true,
       rowData: [],
       frameworkComponents: {
         progressCellRenderer: PostDetailsProgressRendererComponent

vripper-ui/src/app/post-detail/post-details-progress.component.html

@@ -0,0 +1,32 @@
+<div class="container" style="height: 100%; background-color: white">
+  <ng-container *ngIf="postDetails$ | async as details">
+    <div
+      [ngClass]="{
+        error: details.status === 'ERROR'
+      }"
+      class="progress-bar-back"
+      style="position: relative; height: 100%;"
+    >
+      <div
+        class="progress-foreground"
+        fxLayout="row"
+        fxLayoutAlign="space-between"
+        style="background-color: transparent; width: 100%; height: 100%;"
+      >
+        <div class="progress-bar" style="position: absolute; width: 100%; top: 37px; padding: 0 20px">
+          <mat-progress-bar [value]="details.progress" class="example-margin" color="primary" mode="determinate">
+          </mat-progress-bar>
+        </div>
+        <span fxLayout="row">
+          <span style="padding-left: 20px"
+          ><a (click)="goTo()" href="javascript:void(0)" style="color: rgba(0, 0, 0, 0.87);">{{
+              details.url
+            }}</a></span
+          >
+          <span class="filler" fxFlex="grow"></span>
+        </span>
+        <span class="progress-percentage" style="padding-right: 20px">{{ trunc(details.progress) + '%' }}</span>
+      </div>
+    </div>
+  </ng-container>
+</div>

vripper-ui/src/app/post-detail/post-details-progress.component.scss-theme.scss

@@ -0,0 +1,24 @@
+@import '~@angular/material/theming';
+
+@mixin post-detail-renderer-component-theme($theme) {
+  $blue: mat-palette($mat-blue);
+  $red: mat-palette($mat-red);
+
+  .progress-bar .mat-progress-bar-fill::after {
+    background-color: mat-color($blue, 300);
+  }
+
+  .progress-bar .mat-progress-bar-buffer {
+    background-color: mat-color($blue, 100);
+  }
+
+  app-details-cell .error {
+    background-color: mat-color($red, 50);
+    & .progress-bar .mat-progress-bar-fill::after {
+      background-color: mat-color($red, 300);
+    }
+    & .progress-bar .mat-progress-bar-buffer {
+      background-color: mat-color($red, 100);
+    }
+  }
+}

vripper-ui/src/app/post-detail/post-details-progress.component.ts

@@ -1,82 +1,39 @@
 import { WsConnectionService } from '../ws-connection.service';
-import { Component, OnInit, OnDestroy, NgZone } from '@angular/core';
+import {
+  Component,
+  OnInit,
+  OnDestroy,
+  NgZone,
+  ChangeDetectionStrategy,
+  EventEmitter,
+  AfterViewInit
+} from '@angular/core';
 import { AgRendererComponent } from 'ag-grid-angular';
 import { Subscription } from 'rxjs';
 import { PostDetails } from './post-details.model';
 import { WsHandler } from '../ws-handler';
 import { ICellRendererParams } from 'ag-grid-community';
+import { ElectronService } from 'ngx-electron';
 
 @Component({
-  selector: 'app-progress-cell',
-  template: `
-    <div style="height: 100%;">
-      <div
-        class="progress-bar"
-        [ngClass]="{
-          complete: postDetails.status === 'COMPLETE',
-          downloading: postDetails.status === 'DOWNLOADING'
-        }"
-        [style.width]="trunc(postDetails.progress) + '%'"
-      ></div>
-      <div
-        class="progress-bar-back"
-        fxLayout="row"
-        fxLayoutAlign="space-between"
-        [ngClass]="{
-          error: postDetails.status === 'ERROR',
-          stopped: postDetails.status === 'STOPPED',
-          pending: postDetails.status === 'PENDING'
-        }"
-      >
-        <div>{{ postDetails.url }}</div>
-        <div>{{ trunc(postDetails.progress) + '%' }}</div>
-      </div>
-    </div>
-  `,
-  styles: [
-    `
-      .progress-bar-back {
-        position: relative;
-        height: 45px;
-        bottom: 45px;
-        padding: 0 8px;
-        transition: background-color 0.5s;
-      }
-      .progress-bar {
-        background-color: #87a2c7;
-        height: 100%;
-        transition: width 0.5s, background-color 0.5s;
-      }
-      .complete {
-        background-color: #3865a3;
-      }
-      .pending {
-        background-color: white;
-      }
-      .error {
-        background-color: #eb6060;
-      }
-      .downloading {
-        background-color: #87a2c7;
-      }
-      .stopped {
-        background-color: grey;
-      }
-    `
-  ]
+  selector: 'app-details-cell',
+  templateUrl: 'post-details-progress.component.html',
+  styleUrls: ['post-details-progress.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
-export class PostDetailsProgressRendererComponent implements AgRendererComponent, OnInit, OnDestroy {
+export class PostDetailsProgressRendererComponent implements AgRendererComponent, OnInit, OnDestroy, AfterViewInit {
   constructor(
     private wsConnectionService: WsConnectionService,
-    private zone: NgZone
-    ) {
+    private zone: NgZone,
+    public electronService: ElectronService
+  ) {
     this.websocketHandlerPromise = this.wsConnectionService.getConnection();
   }
 
   websocketHandlerPromise: Promise<WsHandler>;
   subscription: Subscription;
-  params: ICellRendererParams;
-  postDetails: PostDetails;
+  postDetails$: EventEmitter<PostDetails> = new EventEmitter();
+  private postDetails: PostDetails;
 
   trunc(value: number): number {
     return Math.trunc(value);
@@ -89,6 +46,7 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
           e.forEach(v => {
             if (this.postDetails.url === v.url) {
               this.postDetails = v;
+              this.postDetails$.emit(this.postDetails);
             }
           });
         });
@@ -96,17 +54,31 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
     });
   }
 
+  goTo() {
+    if (this.electronService.isElectronApp) {
+      this.electronService.shell.openExternal(this.postDetails.url);
+    } else {
+      window.open(this.postDetails.url, '_blank');
+    }
+  }
+
+  ngAfterViewInit(): void {
+    this.postDetails$.emit(this.postDetails);
+  }
+
   ngOnDestroy(): void {
-    this.subscription.unsubscribe();
+    if (this.subscription != null) {
+      this.subscription.unsubscribe();
+    }
   }
 
   agInit(params: ICellRendererParams): void {
-    this.params = params;
     this.postDetails = params.data;
   }
 
   refresh(params: ICellRendererParams): boolean {
     this.postDetails = params.data;
+    this.postDetails$.emit(this.postDetails);
     return true;
   }
 }

vripper-ui/src/app/post-detail/post-details.datasource.ts

@@ -13,7 +13,7 @@ export class PostDetailsDataSource {
     private postId: string,
     private zone: NgZone
     ) {
-    this.websocketHandlerPromise = wsConnectionService.getConnection();
+    this.websocketHandlerPromise = this.wsConnectionService.getConnection();
   }
 
   websocketHandlerPromise: Promise<WsHandler>;