v2.2.5

Fix pixhost and imagezilla to exclude thumbs

.gitignore

@@ -2,4 +2,5 @@
 /**/*/node_modules
 /**/*/target
 /**/*/build-dir
+/**/*/java-runtime
 .idea

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## [2.2.5] - 2019-11-09
+### Changed
+- Exclude thumbs download for imagezilla and pixhost
+- Change retry on fail logic (may help avoid having download errors)
+
+## [2.2.4] - 2019-10-05
+### Added
+- AppImage target for linux
+
+## [2.2.3] - 2019-10-01
+### Changed
+- Strip libjvm.so for debian
+
+## [2.2.2] - 2019-10-01
+### Changed
+- Update product name
+
+## [2.2.1] - 2019-10-01
+### Changed
+- Update dependency list for deb package
+
+## [2.2.0] - 2019-10-01
+### Added
+- Embedd JRE within the app
+### Changed
+- Fix 'Start' on pending items
+- Upgrade to Java 11
+
+## [2.1.0] - 2019-09-30
+### Added
+- Add a create a subfolder per thread option
+
+## [2.0.4] - 2019-09-27
+### Changed
+- Fix some issues with ImxHost
+
 ## [2.0.3] - 2019-09-23
 ### Changed
 - Update deb dependencies

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.0.3</version>
+    <version>2.2.5</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/main.js

@@ -73,13 +73,32 @@ getPort().then(port => {
   ipcMain.on("get-port", event => {
     event.reply("port", port);
   });
-  vripperServer = spawn("java", [
+  let javaBinPath;
+  if(appDir !== undefined) {
+    javaBinPath = path.join(appDir, "java-runtime/bin/java");
+  } else {
+    if(process.platform === 'darwin') {
+      javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+    } else {
+      javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+    }
+  }
+  let jarPath;
+  if(appDir !== undefined) {
+    jarPath = path.join(appDir, "bin/vripper-server.jar");
+  } else {
+    if(process.platform === 'darwin') {
+      jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+    } else {
+      jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+    }
+  }
+  vripperServer = spawn(javaBinPath, [
     "-Xms256m",
     "-Xmx1024m",
     "-Dvripper.server.port=" + port,
     "-jar",
-    appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
-    path.join(app.getPath('exe'), "../bin/vripper-server.jar")
+    jarPath
   ], {
     stdio: 'ignore'
   });

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.0.3",
+  "version": "2.2.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.0.3",
+  "version": "2.2.5",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -8,7 +8,7 @@
   "license": "ISC",
   "build": {
     "appId": "tn.mnlr.vripper",
-    "productName": "V Ripper",
+    "productName": "vripper",
     "files": [
       "**/*",
       "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -31,6 +31,10 @@
       {
         "from": "../vripper-server/target/vripper-server-${version}-electron.jar",
         "to": "bin/vripper-server.jar"
+      },
+      {
+        "from": "java-runtime",
+        "to": "java-runtime"
       }
     ],
     "win": {
@@ -50,7 +54,8 @@
       "packageCategory": "Utility",
       "icon": "icons",
       "target": [
-        "deb"
+        "deb",
+        "AppImage"
       ]
     },
     "deb": {
@@ -60,9 +65,15 @@
         "libnotify4",
         "libappindicator1",
         "libxtst6",
-        "libnss3",
-        "openjdk-8-jre"
+        "libnss3"
       ]
+    },
+    "mac": {
+      "category": "public.app-category.utilities",
+      "target": [
+        "dmg"
+      ],
+      "icon": "icon.png"
     }
   },
   "scripts": {

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.0.3</version>
+    <version>2.2.5</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-electron/pre-build.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
 const fs = require('fs');
 const rimraf = require("rimraf");
 const copydir = require("copy-dir");
-
+const { execSync } = require("child_process");
 
 rimraf.sync('./build/vripper-ui');
 copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
 
 fs.writeFileSync('./build/vripper-ui/index.html', $.html());
 
+console.log('Building runtime environment');
+rimraf.sync('java-runtime');
+execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
+if(process.platform === 'linux') {
+    console.log('Stripping libjvm.so');
+    execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
+}

vripper-server/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>2.0.3</version>
+        <version>2.2.5</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
     <description>vripper-server</description>
 
     <properties>
-        <java.version>1.8</java.version>
+        <java.version>11</java.version>
     </properties>
 
     <dependencies>

vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java

@@ -26,6 +26,8 @@ public class Post {
 
     private String postId;
 
+    private String threadTitle;
+
     private String threadId;
 
     private String title;
@@ -46,7 +48,7 @@ public class Post {
 
     private String forum;
 
-    public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String forum) {
+    public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) {
         this.title = title;
         this.url = url;
         this.images = images;
@@ -54,20 +56,11 @@ public class Post {
         this.postId = postId;
         this.forum = forum;
         this.threadId = threadId;
-        if (this.images.contains(null)) {
-            System.out.println("Oops");
-        }
-        this.images.stream().map(e -> {
-            if (e == null) {
-                System.out.println("Am the cause");
-            }
-            return e.getHost();
-        }).collect(Collectors.toSet());
+        this.threadTitle = threadTitle;
         this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
         this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
         total = images.size();
         status = Status.PENDING;
-
     }
 
     public void setAppStateService(AppStateService appStateService) {

vripper-server/src/main/java/tn/mnlr/vripper/exception/DownloadException.java

@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
         super(message);
     }
 
-    public DownloadException(Exception e) {
+    public DownloadException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class HostException extends Exception {
-    public HostException(Exception e) {
+    public HostException(Throwable e) {
         super(e);
     }
 
@@ -9,7 +9,7 @@ public class HostException extends Exception {
         super(message);
     }
 
-    public HostException(String message, Exception e) {
+    public HostException(String message, Throwable e) {
         super(message,e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HtmlProcessorException.java

@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
 
 public class HtmlProcessorException extends Exception {
 
-    public HtmlProcessorException(Exception e) {
+    public HtmlProcessorException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/PostParseException.java

@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
         super(message);
     }
 
-    public PostParseException(String message, Exception e) {
+    public PostParseException(String message, Throwable e) {
         super(message, e);
     }
 
-    public PostParseException(Exception e) {
+    public PostParseException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/VripperException.java

@@ -5,7 +5,7 @@ public class VripperException extends Exception {
         super(message);
     }
 
-    public VripperException(Exception e) {
+    public VripperException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/XpathException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class XpathException extends Exception {
-    public XpathException(Exception e) {
+    public XpathException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java

@@ -38,6 +38,11 @@ public class AcidimgHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -59,8 +59,10 @@ abstract public class Host {
 
     abstract public String getHost();
 
+    abstract public String getLookup();
+
     public boolean isSupported(String url) {
-        return url.contains(getHost());
+        return url.contains(getLookup());
     }
 
     public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageBamHost.java

@@ -38,6 +38,11 @@ public class ImageBamHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageTwistHost.java

@@ -27,6 +27,11 @@ public class ImageTwistHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java

@@ -17,6 +17,7 @@ public class ImageZillaHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
 
     private static final String host = "imagezilla.net";
+    private static final String lookup = "imagezilla.net/show";
     public static final String IMG_XPATH = "//img[@id='photo']";
 
     @Autowired
@@ -27,6 +28,11 @@ public class ImageZillaHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return lookup;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java

@@ -27,6 +27,11 @@ public class ImgboxHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -41,25 +41,39 @@ public class ImxHost extends Host {
     }
 
     @Override
-    protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
+    public String getLookup() {
+        return host;
+    }
 
-        Document doc = getResponse(url).getDocument();
+    @Override
+    protected void setNameAndUrl(final String _url, final ImageFileData imageFileData) throws HostException {
+
+        String url = _url.replace("http://", "https://");
+        Response resp = getResponse(url);
+        Document doc = resp.getDocument();
 
         Node contDiv;
+        String value = null;
         try {
             logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
+            Node node = contDiv.getAttributes().getNamedItem("value");
+            if (node != null) {
+                value = node.getTextContent();
+            }
         } catch (XpathException e) {
             throw new HostException(e);
         }
 
         if (contDiv != null) {
+            if (value == null) {
+                throw new HostException("Failed to obtain value attribute from continue input");
+            }
             logger.info(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
             HttpPost httpPost = cm.buildHttpPost(url);
-            httpPost.addHeader("Referer", url);
             List<NameValuePair> params = new ArrayList<>();
-            params.add(new BasicNameValuePair("imgContinue", "Continue to image ... "));
+            params.add(new BasicNameValuePair("imgContinue", value));
             try {
                 httpPost.setEntity(new UrlEncodedFormEntity(params));
             } catch (Exception e) {

vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java

@@ -17,6 +17,7 @@ public class PixhostHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
 
     private static final String host = "pixhost.to";
+    private static final String lookup = "pixhost.to/show";
     public static final String IMG_XPATH = "//img[@id='image']";
 
     @Autowired
@@ -27,6 +28,11 @@ public class PixhostHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return lookup;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/TurboImageHost.java

@@ -28,6 +28,11 @@ public class TurboImageHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.AppStateService;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
-import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -53,8 +53,8 @@ public class ExecutionService {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withDelay(Duration.ofSeconds(5))
-                .withMaxRetries(2)
+                .withBackoff(10, 60, ChronoUnit.SECONDS)
+                .withMaxRetries(4)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -37,6 +37,7 @@ public class AppSettingsService {
     private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
     private final String FORCE_ORDER = "FORCE_ORDER";
     private final String SUBFOLDER = "SUBFOLDER";
+    private final String THREADSUBFOLDER = "THREADSUBFOLDER";
     private final String CLEAR = "CLEAR";
     private final String DARK_THEME = "DARK_THEME";
 
@@ -49,6 +50,7 @@ public class AppSettingsService {
     private boolean vThanks;
     private boolean desktopClipboard;
     private boolean subLocation;
+    private boolean threadSubLocation;
     private boolean forceOrder;
     private boolean clearCompleted;
     private boolean darkTheme;
@@ -73,6 +75,7 @@ public class AppSettingsService {
         desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
         forceOrder = prefs.getBoolean(FORCE_ORDER, false);
         subLocation = prefs.getBoolean(SUBFOLDER, false);
+        threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
         clearCompleted = prefs.getBoolean(CLEAR, false);
         darkTheme = prefs.getBoolean(DARK_THEME, false);
     }
@@ -90,6 +93,7 @@ public class AppSettingsService {
         prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
         prefs.putBoolean(FORCE_ORDER, forceOrder);
         prefs.putBoolean(SUBFOLDER, subLocation);
+        prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
         prefs.putBoolean(CLEAR, clearCompleted);
         prefs.putBoolean(DARK_THEME, darkTheme);
 
@@ -163,10 +167,12 @@ public class AppSettingsService {
         private boolean forceOrder;
         @JsonProperty("subLocation")
         private boolean subLocation;
+        @JsonProperty("threadSubLocation")
+        private boolean threadSubLocation;
         @JsonProperty("clearCompleted")
         private boolean clearCompleted;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean clearCompleted) {
+        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
             this.autoStart = autoStart;
@@ -177,6 +183,7 @@ public class AppSettingsService {
             this.desktopClipboard = desktopClipboard;
             this.forceOrder = forceOrder;
             this.subLocation = subLocation;
+            this.threadSubLocation = threadSubLocation;
             this.clearCompleted = clearCompleted;
         }
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -7,6 +7,7 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.springframework.stereotype.Service;
 
@@ -38,6 +39,7 @@ public class ConnectionManager {
     public HttpClientBuilder getClient() {
         return HttpClients.custom()
                 .setConnectionManager(pcm)
+                .setRedirectStrategy(new LaxRedirectStrategy())
                 .setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
                 .setDefaultRequestConfig(rc);
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/PathService.java

@@ -22,7 +22,9 @@ public class PathService {
     public final File getDownloadDestinationFolder(String postId) {
         Post post = appStateService.getCurrentPosts().get(postId);
         String postTitle = post.getTitle();
+        String threadTitle = post.getThreadTitle();
         File sourceFolder = appSettingsService.isSubLocation() ? new File(appSettingsService.getDownloadPath(), sanitize(post.getForum())) : new File(appSettingsService.getDownloadPath());
+        sourceFolder = appSettingsService.isThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
         return new File(sourceFolder, sanitize(postTitle + "_" + postId));
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -2,6 +2,8 @@ package tn.mnlr.vripper.services;
 
 import io.reactivex.processors.PublishProcessor;
 import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -21,14 +23,18 @@ import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.q.DownloadQ;
 
+import javax.annotation.PostConstruct;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParserFactory;
 import java.io.BufferedInputStream;
+import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 @Service
 public class PostParser {
@@ -60,9 +66,22 @@ public class PostParser {
     @Autowired
     private VipergirlsAuthService vipergirlsAuthService;
 
+    private RetryPolicy<Object> retryPolicy;
+
     public PostParser() throws ParserConfigurationException, SAXException {
     }
 
+    @PostConstruct
+    private void init() {
+
+        retryPolicy = new RetryPolicy<>()
+                .handleIf(e -> e instanceof IOException)
+                .withBackoff(2, 20, ChronoUnit.SECONDS)
+                .withMaxRetries(4)
+                .abortOn(InterruptedException.class)
+                .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+    }
+
     public void addPost(String postId, String threadId) throws PostParseException {
 
         if (appStateService.getCurrentPosts().containsKey(postId)) {
@@ -161,19 +180,32 @@ public class PostParser {
             } catch (URISyntaxException e) {
                 throw new PostParseException(e);
             }
+
             VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
             HttpClient connection = cm.getClient().build();
+            AtomicReference<Throwable> thr = new AtomicReference<>();
             logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
+            Failsafe.with(retryPolicy)
+                    .onFailure(e -> thr.set(e.getFailure()))
+                    .get(() -> {
+                        try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
+                            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                                throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                            }
 
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
+                            try {
+                                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                            } catch (Exception e) {
+                                throw new PostParseException(String.format("Failed to parse thread %s", threadId), e);
+                            } finally {
+                                EntityUtils.consumeQuietly(response.getEntity());
+                            }
+                        }
+                        return null;
+                    });
+            if (thr.get() != null) {
+                logger.error(String.format("parsing failed for thread %s", threadId), thr.get());
+                throw new PostParseException(thr.get());
             }
             return null;
         }
@@ -265,7 +297,6 @@ public class PostParser {
 
         public Post parse() throws PostParseException {
 
-            Post post;
             logger.info(String.format("Parsing post %s", postId));
             HttpGet httpGet;
             try {
@@ -280,18 +311,29 @@ public class PostParser {
             }
             VRPostHandler handler = new VRPostHandler(threadId, postId);
             HttpClient connection = cm.getClient().build();
+            AtomicReference<Throwable> thr = new AtomicReference<>();
             logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
+            Post post = Failsafe.with(retryPolicy)
+                    .onFailure(e -> thr.set(e.getFailure()))
+                    .get(() -> {
+                        try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
+                            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                                throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                            }
 
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                post = handler.getParsedPost();
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
+                            try {
+                                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                                return handler.getParsedPost();
+                            } catch (Exception e) {
+                                throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
+                            } finally {
+                                EntityUtils.consumeQuietly(response.getEntity());
+                            }
+                        }
+                    });
+            if (thr.get() != null || post == null) {
+                logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
+                throw new PostParseException(thr.get());
             }
             return post;
         }
@@ -376,6 +418,7 @@ public class PostParser {
                                 metadata,
                                 postId,
                                 threadId,
+                                threadTitle,
                                 forum
                         );
                     }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -75,6 +75,7 @@ public class SettingsRestEndpoint {
             this.settings.setDesktopClipboard(settings.isDesktopClipboard());
             this.settings.setForceOrder(settings.isForceOrder());
             this.settings.setSubLocation(settings.isSubLocation());
+            this.settings.setThreadSubLocation(settings.isThreadSubLocation());
             this.settings.setClearCompleted(settings.isClearCompleted());
 
             this.settings.save();
@@ -99,6 +100,7 @@ public class SettingsRestEndpoint {
                 settings.isDesktopClipboard(),
                 settings.isForceOrder(),
                 settings.isSubLocation(),
+                settings.isThreadSubLocation(),
                 settings.isClearCompleted()
         );
     }

vripper-server/src/main/resources/application.properties

@@ -6,4 +6,5 @@ server.port=${vripper.server.port:8080}
 management.endpoints.web.exposure.include=shutdown
 management.endpoint.shutdown.enabled=true
 endpoints.shutdown.enabled=true
-spring.profiles.active=portable
+#spring.profiles.active=portable
+spring.profiles.active=installer

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.0.3",
+  "version": "2.2.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.0.3",
+  "version": "2.2.5",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.0.3</version>
+    <version>2.2.5</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.component.scss

@@ -1,11 +1,3 @@
-#logo {
-  height: 48px;
-  margin-right: 10px;
-  background-image: url('../assets/logo.png');
-  background-repeat: no-repeat;
-  background-size: 48px 48px;
-}
-
 #title {
   user-select: none
 }

vripper-ui/src/app/posts/post-progress.renderer.component.html

@@ -42,7 +42,6 @@
           <button
             (click)="restart()"
             *ngIf="
-              postState.status === 'PENDING' ||
               (postState.status === 'COMPLETE' && postState.progress !== 100) ||
               postState.status === 'ERROR' ||
               postState.status === 'STOPPED'
@@ -54,7 +53,7 @@
           </button>
           <button
             (click)="stop()"
-            *ngIf="postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL'"
+            *ngIf="postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL' || postState.status === 'PENDING'"
             mat-menu-item
           >
             <ng-container>

vripper-ui/src/app/settings/settings.component.html

@@ -42,6 +42,11 @@
             >Save posts in sub folders
             </mat-checkbox>
 
+            <mat-checkbox *ngIf="generalSettingsForm.get('subLocation').value" color="primary"
+                          formControlName="threadSubLocation" name="threadSubLocation" style="margin-left: 15px"
+            >Create a subfolder per thread
+            </mat-checkbox>
+
             <mat-checkbox color="primary" formControlName="forceOrder" name="forceOrder"
             >Force image ordering (prepend incremental numbers)
             </mat-checkbox>

vripper-ui/src/app/settings/settings.component.ts

@@ -30,6 +30,7 @@ export class SettingsComponent implements OnInit {
     autoStart: new FormControl(false),
     forceOrder: new FormControl(false),
     subLocation: new FormControl(false),
+    threadSubLocation: new FormControl(false),
     clearCompleted: new FormControl(false),
     vLogin: new FormControl(false),
     vUsername: new FormControl(''),