v2.2.5

Fix pixhost and imagezilla to exclude thumbs

.gitignore

@@ -2,4 +2,5 @@
 /**/*/node_modules
 /**/*/target
 /**/*/build-dir
+/**/*/java-runtime
 .idea

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## [2.2.5] - 2019-11-09
+### Changed
+- Exclude thumbs download for imagezilla and pixhost
+- Change retry on fail logic (may help avoid having download errors)
+
+## [2.2.4] - 2019-10-05
+### Added
+- AppImage target for linux
+
+## [2.2.3] - 2019-10-01
+### Changed
+- Strip libjvm.so for debian
+
+## [2.2.2] - 2019-10-01
+### Changed
+- Update product name
+
+## [2.2.1] - 2019-10-01
+### Changed
+- Update dependency list for deb package
+
+## [2.2.0] - 2019-10-01
+### Added
+- Embedd JRE within the app
+### Changed
+- Fix 'Start' on pending items
+- Upgrade to Java 11
+
 ## [2.1.0] - 2019-09-30
 ### Added
 - Add a create a subfolder per thread option

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.1.0</version>
+    <version>2.2.5</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/main.js

@@ -73,13 +73,32 @@ getPort().then(port => {
   ipcMain.on("get-port", event => {
     event.reply("port", port);
   });
-  vripperServer = spawn("java", [
+  let javaBinPath;
+  if(appDir !== undefined) {
+    javaBinPath = path.join(appDir, "java-runtime/bin/java");
+  } else {
+    if(process.platform === 'darwin') {
+      javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
+    } else {
+      javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
+    }
+  }
+  let jarPath;
+  if(appDir !== undefined) {
+    jarPath = path.join(appDir, "bin/vripper-server.jar");
+  } else {
+    if(process.platform === 'darwin') {
+      jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
+    } else {
+      jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
+    }
+  }
+  vripperServer = spawn(javaBinPath, [
     "-Xms256m",
     "-Xmx1024m",
     "-Dvripper.server.port=" + port,
     "-jar",
-    appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
-    path.join(app.getPath('exe'), "../bin/vripper-server.jar")
+    jarPath
   ], {
     stdio: 'ignore'
   });

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.1.0",
+  "version": "2.2.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "2.1.0",
+  "version": "2.2.5",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -8,7 +8,7 @@
   "license": "ISC",
   "build": {
     "appId": "tn.mnlr.vripper",
-    "productName": "V Ripper",
+    "productName": "vripper",
     "files": [
       "**/*",
       "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -31,6 +31,10 @@
       {
         "from": "../vripper-server/target/vripper-server-${version}-electron.jar",
         "to": "bin/vripper-server.jar"
+      },
+      {
+        "from": "java-runtime",
+        "to": "java-runtime"
       }
     ],
     "win": {
@@ -50,7 +54,8 @@
       "packageCategory": "Utility",
       "icon": "icons",
       "target": [
-        "deb"
+        "deb",
+        "AppImage"
       ]
     },
     "deb": {
@@ -60,9 +65,15 @@
         "libnotify4",
         "libappindicator1",
         "libxtst6",
-        "libnss3",
-        "openjdk-8-jre"
+        "libnss3"
       ]
+    },
+    "mac": {
+      "category": "public.app-category.utilities",
+      "target": [
+        "dmg"
+      ],
+      "icon": "icon.png"
     }
   },
   "scripts": {

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.1.0</version>
+    <version>2.2.5</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-electron/pre-build.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
 const fs = require('fs');
 const rimraf = require("rimraf");
 const copydir = require("copy-dir");
-
+const { execSync } = require("child_process");
 
 rimraf.sync('./build/vripper-ui');
 copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
 
 fs.writeFileSync('./build/vripper-ui/index.html', $.html());
 
+console.log('Building runtime environment');
+rimraf.sync('java-runtime');
+execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
+if(process.platform === 'linux') {
+    console.log('Stripping libjvm.so');
+    execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
+}

vripper-server/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>2.1.0</version>
+        <version>2.2.5</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
     <description>vripper-server</description>
 
     <properties>
-        <java.version>1.8</java.version>
+        <java.version>11</java.version>
     </properties>
 
     <dependencies>

vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java

@@ -57,20 +57,10 @@ public class Post {
         this.forum = forum;
         this.threadId = threadId;
         this.threadTitle = threadTitle;
-        if (this.images.contains(null)) {
-            System.out.println("Oops");
-        }
-        this.images.stream().map(e -> {
-            if (e == null) {
-                System.out.println("Am the cause");
-            }
-            return e.getHost();
-        }).collect(Collectors.toSet());
         this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
         this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
         total = images.size();
         status = Status.PENDING;
-
     }
 
     public void setAppStateService(AppStateService appStateService) {

vripper-server/src/main/java/tn/mnlr/vripper/exception/DownloadException.java

@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
         super(message);
     }
 
-    public DownloadException(Exception e) {
+    public DownloadException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class HostException extends Exception {
-    public HostException(Exception e) {
+    public HostException(Throwable e) {
         super(e);
     }
 
@@ -9,7 +9,7 @@ public class HostException extends Exception {
         super(message);
     }
 
-    public HostException(String message, Exception e) {
+    public HostException(String message, Throwable e) {
         super(message,e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/HtmlProcessorException.java

@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
 
 public class HtmlProcessorException extends Exception {
 
-    public HtmlProcessorException(Exception e) {
+    public HtmlProcessorException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/PostParseException.java

@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
         super(message);
     }
 
-    public PostParseException(String message, Exception e) {
+    public PostParseException(String message, Throwable e) {
         super(message, e);
     }
 
-    public PostParseException(Exception e) {
+    public PostParseException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/VripperException.java

@@ -5,7 +5,7 @@ public class VripperException extends Exception {
         super(message);
     }
 
-    public VripperException(Exception e) {
+    public VripperException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/exception/XpathException.java

@@ -1,7 +1,7 @@
 package tn.mnlr.vripper.exception;
 
 public class XpathException extends Exception {
-    public XpathException(Exception e) {
+    public XpathException(Throwable e) {
         super(e);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java

@@ -38,6 +38,11 @@ public class AcidimgHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -59,8 +59,10 @@ abstract public class Host {
 
     abstract public String getHost();
 
+    abstract public String getLookup();
+
     public boolean isSupported(String url) {
-        return url.contains(getHost());
+        return url.contains(getLookup());
     }
 
     public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageBamHost.java

@@ -38,6 +38,11 @@ public class ImageBamHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageTwistHost.java

@@ -27,6 +27,11 @@ public class ImageTwistHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java

@@ -17,6 +17,7 @@ public class ImageZillaHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
 
     private static final String host = "imagezilla.net";
+    private static final String lookup = "imagezilla.net/show";
     public static final String IMG_XPATH = "//img[@id='photo']";
 
     @Autowired
@@ -27,6 +28,11 @@ public class ImageZillaHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return lookup;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java

@@ -27,6 +27,11 @@ public class ImgboxHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -40,6 +40,11 @@ public class ImxHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String _url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java

@@ -17,6 +17,7 @@ public class PixhostHost extends Host {
     private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
 
     private static final String host = "pixhost.to";
+    private static final String lookup = "pixhost.to/show";
     public static final String IMG_XPATH = "//img[@id='image']";
 
     @Autowired
@@ -27,6 +28,11 @@ public class PixhostHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return lookup;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/host/TurboImageHost.java

@@ -28,6 +28,11 @@ public class TurboImageHost extends Host {
         return host;
     }
 
+    @Override
+    public String getLookup() {
+        return host;
+    }
+
     @Override
     protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
 

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.AppStateService;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
-import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -53,8 +53,8 @@ public class ExecutionService {
 
         retryPolicy = new RetryPolicy<>()
                 .handleIf(e -> !(e instanceof InterruptedException))
-                .withDelay(Duration.ofSeconds(5))
-                .withMaxRetries(2)
+                .withBackoff(10, 60, ChronoUnit.SECONDS)
+                .withMaxRetries(4)
                 .abortOn(InterruptedException.class)
                 .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
 

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -2,6 +2,8 @@ package tn.mnlr.vripper.services;
 
 import io.reactivex.processors.PublishProcessor;
 import lombok.Getter;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.RetryPolicy;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -21,14 +23,18 @@ import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.q.DownloadQ;
 
+import javax.annotation.PostConstruct;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParserFactory;
 import java.io.BufferedInputStream;
+import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 @Service
 public class PostParser {
@@ -60,9 +66,22 @@ public class PostParser {
     @Autowired
     private VipergirlsAuthService vipergirlsAuthService;
 
+    private RetryPolicy<Object> retryPolicy;
+
     public PostParser() throws ParserConfigurationException, SAXException {
     }
 
+    @PostConstruct
+    private void init() {
+
+        retryPolicy = new RetryPolicy<>()
+                .handleIf(e -> e instanceof IOException)
+                .withBackoff(2, 20, ChronoUnit.SECONDS)
+                .withMaxRetries(4)
+                .abortOn(InterruptedException.class)
+                .onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
+    }
+
     public void addPost(String postId, String threadId) throws PostParseException {
 
         if (appStateService.getCurrentPosts().containsKey(postId)) {
@@ -161,19 +180,32 @@ public class PostParser {
             } catch (URISyntaxException e) {
                 throw new PostParseException(e);
             }
+
             VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
             HttpClient connection = cm.getClient().build();
+            AtomicReference<Throwable> thr = new AtomicReference<>();
             logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
+            Failsafe.with(retryPolicy)
+                    .onFailure(e -> thr.set(e.getFailure()))
+                    .get(() -> {
+                        try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
+                            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                                throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                            }
 
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
+                            try {
+                                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                            } catch (Exception e) {
+                                throw new PostParseException(String.format("Failed to parse thread %s", threadId), e);
+                            } finally {
+                                EntityUtils.consumeQuietly(response.getEntity());
+                            }
+                        }
+                        return null;
+                    });
+            if (thr.get() != null) {
+                logger.error(String.format("parsing failed for thread %s", threadId), thr.get());
+                throw new PostParseException(thr.get());
             }
             return null;
         }
@@ -265,7 +297,6 @@ public class PostParser {
 
         public Post parse() throws PostParseException {
 
-            Post post;
             logger.info(String.format("Parsing post %s", postId));
             HttpGet httpGet;
             try {
@@ -280,18 +311,29 @@ public class PostParser {
             }
             VRPostHandler handler = new VRPostHandler(threadId, postId);
             HttpClient connection = cm.getClient().build();
+            AtomicReference<Throwable> thr = new AtomicReference<>();
             logger.info(String.format("Requesting %s", httpGet));
-            try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
-                if (response.getStatusLine().getStatusCode() / 100 != 2) {
-                    throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
-                }
+            Post post = Failsafe.with(retryPolicy)
+                    .onFailure(e -> thr.set(e.getFailure()))
+                    .get(() -> {
+                        try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
+                            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+                                throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
+                            }
 
-                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
-                post = handler.getParsedPost();
-                EntityUtils.consumeQuietly(response.getEntity());
-            } catch (Exception e) {
-                logger.error("parsing failed", e);
-                throw new PostParseException(e);
+                            try {
+                                factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
+                                return handler.getParsedPost();
+                            } catch (Exception e) {
+                                throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
+                            } finally {
+                                EntityUtils.consumeQuietly(response.getEntity());
+                            }
+                        }
+                    });
+            if (thr.get() != null || post == null) {
+                logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
+                throw new PostParseException(thr.get());
             }
             return post;
         }

vripper-server/src/main/resources/application.properties

@@ -6,4 +6,5 @@ server.port=${vripper.server.port:8080}
 management.endpoints.web.exposure.include=shutdown
 management.endpoint.shutdown.enabled=true
 endpoints.shutdown.enabled=true
-spring.profiles.active=portable
+#spring.profiles.active=portable
+spring.profiles.active=installer

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.1.0",
+  "version": "2.2.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "2.1.0",
+  "version": "2.2.5",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>2.1.0</version>
+    <version>2.2.5</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.component.scss

@@ -1,11 +1,3 @@
-#logo {
-  height: 48px;
-  margin-right: 10px;
-  background-image: url('../assets/logo.png');
-  background-repeat: no-repeat;
-  background-size: 48px 48px;
-}
-
 #title {
   user-select: none
 }

vripper-ui/src/app/posts/post-progress.renderer.component.html

@@ -42,7 +42,6 @@
           <button
             (click)="restart()"
             *ngIf="
-              postState.status === 'PENDING' ||
               (postState.status === 'COMPLETE' && postState.progress !== 100) ||
               postState.status === 'ERROR' ||
               postState.status === 'STOPPED'
@@ -54,7 +53,7 @@
           </button>
           <button
             (click)="stop()"
-            *ngIf="postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL'"
+            *ngIf="postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL' || postState.status === 'PENDING'"
             mat-menu-item
           >
             <ng-container>