v1.4.9

* Fixed a bug when creating image file names
* Remove unnecessary logs
* Fix data.json init

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [1.4.9] - 2019-08-13
+### Changed
+- Update Built-By to be generic
+
+## [1.4.8] - 2019-08-13
+### Changed
+- Add option to force image ordering
+
+## [1.4.7] - 2019-08-13
+### Changed
+- Fixed a bug in image filename extensions 
+
+## [1.4.6] - 2019-08-06
+### Changed
+- Fixed a bug when creating image file names
+- Remove unnecessary logs
+- Fix data.json init 
+
 ## [1.4.5] - 2019-08-03
 ### Changed
 - Add min width and height for electron app

README.md

@@ -0,0 +1,26 @@
+# Viper Ripper!
+
+This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
+The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
+
+## How to build
+You need a recent version of maven 3.6.1+.
+
+The application support 2 build modes:
+ - Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
+ - Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
+
+Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
+
+To build the server app:
+
+    mvn clean install
+To build the desktop app:
+
+    mvn clean install -Pelectron
+
+Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
+
+*Partially cross platform:
+Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
+The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.5</version>
+    <version>1.4.9</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "1.4.5",
+  "version": "1.4.9",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "1.4.5",
+  "version": "1.4.9",
   "description": "",
   "main": "main.js",
   "author": "",

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-      <version>1.4.5</version>
+    <version>1.4.9</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>1.4.5</version>
+        <version>1.4.9</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
@@ -129,6 +129,7 @@
                     <archive>
                         <manifestEntries>
                             <Build-Time>${maven.build.timestamp}</Build-Time>
+                            <Built-By>${user.name}</Built-By>
                         </manifestEntries>
                         <manifestSections>
                             <manifestSection>

vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java

@@ -9,7 +9,6 @@ import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.Objects;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 @Getter
@@ -30,6 +29,8 @@ public class Image {
 
     private String url;
 
+    private int index;
+
     private AtomicLong current = new AtomicLong(0);
     private Status status;
     private BehaviorProcessor<Image> imageStateProcessor;
@@ -37,12 +38,13 @@ public class Image {
     @Setter
     private long total = 0;
 
-    public Image(String url, String postId, String postName, Host host, AppStateService appStateService) {
+    public Image(String url, String postId, String postName, Host host, AppStateService appStateService, int index) {
         this.url = url;
         this.postId = postId;
         this.postName = postName;
         this.host = host;
         this.appStateService = appStateService;
+        this.index = index;
         status = Status.PENDING;
         imageStateProcessor = BehaviorProcessor.create();
         appStateService.getCurrentImages().put(this.url, this);

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -18,10 +18,14 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
 import tn.mnlr.vripper.q.ImageFileData;
 import tn.mnlr.vripper.services.*;
 
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Iterator;
 import java.util.Random;
 
 @Service
@@ -72,17 +76,12 @@ abstract public class Host {
              * END HOST SPECIFIC
              */
 
-            if (!imageFileData.getImageName().toLowerCase().endsWith(".jpg") && !imageFileData.getImageName().toLowerCase().endsWith(".jpeg")) {
-                imageFileData.setImageName(imageFileData.getImageName() + ".jpg");
-            }
-
+            imageFileData.setImageName(formatImageFileName(imageFileData.getImageName()));
             File destinationFolder = new File(appSettingsService.getDownloadPath(), sanitize(image.getPostName() + "_" + image.getPostId()));
             logger.info(String.format("Saving to %s", destinationFolder.getPath()));
             if (!destinationFolder.exists()) {
                 logger.info(String.format("Creating %s", destinationFolder.getPath()));
                 destinationFolder.mkdirs();
-            } else {
-                logger.warn(String.format("%s already exists, the file will be overridden", destinationFolder.getPath()));
             }
 
             HttpClient client = cm.getClient().build();
@@ -95,9 +94,10 @@ abstract public class Host {
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
+                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
                 try (
                         InputStream downloadStream = response.getEntity().getContent();
-                        FileOutputStream fos = new FileOutputStream(destinationFolder.getPath() + File.separator + imageFileData.getImageName())
+                        FileOutputStream fos = new FileOutputStream(outputFile)
                 ) {
 
                     image.setTotal(response.getEntity().getContentLength());
@@ -112,6 +112,7 @@ abstract public class Host {
                         downloadSpeedService.increase(read);
                     }
                     EntityUtils.consumeQuietly(response.getEntity());
+                    checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
                 }
             }
         } catch (Exception e) {
@@ -122,6 +123,41 @@ abstract public class Host {
         }
     }
 
+    private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
+        try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
+            Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
+            if (!it.hasNext()) {
+                throw new HostException("Image file is not recognized!");
+            }
+            ImageReader reader = it.next();
+            String formatName = reader.getFormatName();
+            if (formatName.toUpperCase().equals("JPEG")) {
+                formatName = "jpg";
+            }
+            String outImageName = (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase();
+            outputFile.renameTo(new File(outputFile.getParent(), outImageName));
+        } catch (Exception e) {
+            throw new HostException("Failed to rename output file", e);
+        }
+    }
+
+    /**
+     * Will sanitize the image name and remove extension
+     *
+     * @param imageName
+     * @return
+     */
+    protected String formatImageFileName(String imageName) {
+        int extensionIndex = imageName.lastIndexOf('.');
+        String fileName;
+        if (extensionIndex != -1) {
+            fileName = imageName.substring(0, extensionIndex);
+        } else {
+            fileName = imageName;
+        }
+        return sanitize(fileName);
+    }
+
     /**
      * Just for testing, you may ignore
      * @throws Exception

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -34,6 +34,7 @@ public class AppSettingsService {
     private final String V_PASSWORD = "VPASSWORD";
     private final String V_THANKS = "VTHANKS";
     private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
+    private final String FORCE_ORDER = "FORCE_ORDER";
 
     private String downloadPath;
     private int maxThreads;
@@ -43,6 +44,7 @@ public class AppSettingsService {
     private String vPassword;
     private boolean vThanks;
     private boolean desktopClipboard;
+    private boolean forceOrder;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -62,6 +64,7 @@ public class AppSettingsService {
         vPassword = prefs.get(V_PASSWORD, "");
         vThanks = prefs.getBoolean(V_THANKS, false);
         desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
+        forceOrder = prefs.getBoolean(FORCE_ORDER, false);
     }
 
     @PreDestroy
@@ -75,6 +78,7 @@ public class AppSettingsService {
         prefs.put(V_PASSWORD, vPassword);
         prefs.putBoolean(V_THANKS, vThanks);
         prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
+        prefs.putBoolean(FORCE_ORDER, forceOrder);
 
         try {
             prefs.sync();
@@ -121,8 +125,10 @@ public class AppSettingsService {
         private boolean vThanks;
         @JsonProperty("desktopClipboard")
         private boolean desktopClipboard;
+        @JsonProperty("forceOrder")
+        private boolean forceOrder;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard) {
+        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
             this.autoStart = autoStart;
@@ -131,6 +137,7 @@ public class AppSettingsService {
             this.vPassword = vPassword;
             this.vThanks = vThanks;
             this.desktopClipboard = desktopClipboard;
+            this.forceOrder = forceOrder;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -45,8 +45,8 @@ public class AppStateService {
         }
     }
 
-    public Image createImage(String pageUrl, String postId, String postName, Host host) {
-        return new Image(pageUrl, postId, postName, host, this);
+    public Image createImage(String pageUrl, String postId, String postName, Host host, int index) {
+        return new Image(pageUrl, postId, postName, host, this, index);
     }
 
     public Post createPost(String title, String url, List<Image> images, Map<String, String> metadata, String postId, String postCounter) {

vripper-server/src/main/java/tn/mnlr/vripper/services/DownloadSpeedService.java

@@ -21,7 +21,7 @@ public class DownloadSpeedService {
         this.read.addAndGet(read);
     }
 
-    @Scheduled(fixedRate = 1000)
+    @Scheduled(fixedDelay = 1000)
     private void calc() {
         readBytesPerSecond.onNext(read.getAndSet(0));
     }

vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java

@@ -16,9 +16,10 @@ import tn.mnlr.vripper.entities.mixin.persistance.PostPersistanceMixin;
 
 import javax.annotation.PreDestroy;
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
@@ -48,6 +49,9 @@ public class PersistenceService {
             try {
                 if(dataFile.createNewFile()) {
                     logger.info("Data file successfully created");
+                    try (FileWriter fw = new FileWriter(dataFile)) {
+                        fw.write("{}");
+                    }
                 } else {
                     logger.info("Data file already exists");
                 }
@@ -100,7 +104,7 @@ public class PersistenceService {
 
         String jsonContent;
         try {
-            jsonContent = Files.readAllLines(Paths.get(VripperApplication.dataPath), Charset.forName("UTF-8")).stream().collect(Collectors.joining());
+            jsonContent = Files.readAllLines(Paths.get(VripperApplication.dataPath), StandardCharsets.UTF_8).stream().collect(Collectors.joining());
         } catch (Exception e) {
             logger.warn("data file not found, previous state cannot be restored", e);
             return;

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -180,7 +180,7 @@ public class PostParser {
                     }
                     if (foundHost != null) {
                         logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref));
-                        imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost));
+                        imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost, imagesList.size() + 1));
                     } else {
                         logger.warn(String.format("unsupported host for %s, skipping", imageHref));
                         continue;

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -7,8 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.exception.ValidationException;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.VipergirlsAuthService;
 
 @RestController
@@ -57,6 +57,7 @@ public class SettingsRestEndpoint {
             this.settings.setVThanks(false);
         }
         this.settings.setDesktopClipboard(settings.isDesktopClipboard());
+        this.settings.setForceOrder(settings.isForceOrder());
 
         this.settings.save();
 
@@ -76,7 +77,8 @@ public class SettingsRestEndpoint {
                 settings.getVUsername(),
                 settings.getVPassword(),
                 settings.isVThanks(),
-                settings.isDesktopClipboard()
+                settings.isDesktopClipboard(),
+                settings.isForceOrder()
         );
     }
 

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "1.4.5",
+  "version": "1.4.9",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "1.4.5",
+  "version": "1.4.9",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.5</version>
+    <version>1.4.9</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/settings/settings.component.html

@@ -35,6 +35,11 @@
               />
             </mat-form-field>
 
+            <mat-checkbox color="primary" formControlName="forceOrder" name="forceOrder"
+            >Force image ordering (prepend incremental numbers)
+            </mat-checkbox
+            >
+
             <mat-checkbox color="primary" formControlName="autoStart" name="autoStart"
               >Auto start downloads</mat-checkbox
             >

vripper-ui/src/app/settings/settings.component.scss

@@ -18,3 +18,7 @@
 .container form section div > * {
   width: 100%;
 }
+
+mat-checkbox {
+  display: block;
+}

vripper-ui/src/app/settings/settings.component.ts

@@ -25,6 +25,7 @@ export class SettingsComponent implements OnInit {
     downloadPath: new FormControl(''),
     maxThreads: new FormControl(''),
     autoStart: new FormControl(false),
+    forceOrder: new FormControl(false),
     vLogin: new FormControl(false),
     vUsername: new FormControl(''),
     vPassword: new FormControl(''),