Merge branch 'master' of https://github.com/death-claw/vripper-project into development

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [1.4.10] - 2019-08-15
+### Changed
+- Fix a rename issue on windows
+
+## [1.4.9] - 2019-08-13
+### Changed
+- Update Built-By to be generic
+
+## [1.4.8] - 2019-08-13
+### Changed
+- Add option to force image ordering
+
+## [1.4.7] - 2019-08-13
+### Changed
+- Fixed a bug in image filename extensions 
+
 ## [1.4.6] - 2019-08-06
 ### Changed
 - Fixed a bug when creating image file names

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.6</version>
+    <version>1.4.10</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "1.4.6",
+  "version": "1.4.10",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "1.4.6",
+  "version": "1.4.10",
   "description": "",
   "main": "main.js",
   "author": "",

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.6</version>
+    <version>1.4.10</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>1.4.6</version>
+        <version>1.4.10</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>
@@ -129,6 +129,7 @@
                     <archive>
                         <manifestEntries>
                             <Build-Time>${maven.build.timestamp}</Build-Time>
+                            <Built-By>${user.name}</Built-By>
                         </manifestEntries>
                         <manifestSections>
                             <manifestSection>

vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java

@@ -9,7 +9,6 @@ import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.services.AppStateService;
 
 import java.util.Objects;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 @Getter
@@ -30,6 +29,8 @@ public class Image {
 
     private String url;
 
+    private int index;
+
     private AtomicLong current = new AtomicLong(0);
     private Status status;
     private BehaviorProcessor<Image> imageStateProcessor;
@@ -37,12 +38,13 @@ public class Image {
     @Setter
     private long total = 0;
 
-    public Image(String url, String postId, String postName, Host host, AppStateService appStateService) {
+    public Image(String url, String postId, String postName, Host host, AppStateService appStateService, int index) {
         this.url = url;
         this.postId = postId;
         this.postName = postName;
         this.host = host;
         this.appStateService = appStateService;
+        this.index = index;
         status = Status.PENDING;
         imageStateProcessor = BehaviorProcessor.create();
         appStateService.getCurrentImages().put(this.url, this);

vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java

@@ -18,10 +18,15 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
 import tn.mnlr.vripper.q.ImageFileData;
 import tn.mnlr.vripper.services.*;
 
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.Iterator;
 import java.util.Random;
 
 @Service
@@ -72,10 +77,7 @@ abstract public class Host {
              * END HOST SPECIFIC
              */
 
-            if (!imageFileData.getImageName().toLowerCase().endsWith(".jpg") && !imageFileData.getImageName().toLowerCase().endsWith(".jpeg")) {
-                imageFileData.setImageName(imageFileData.getImageName() + ".jpg");
-            }
-
+            imageFileData.setImageName(formatImageFileName(imageFileData.getImageName()));
             File destinationFolder = new File(appSettingsService.getDownloadPath(), sanitize(image.getPostName() + "_" + image.getPostId()));
             logger.info(String.format("Saving to %s", destinationFolder.getPath()));
             if (!destinationFolder.exists()) {
@@ -93,11 +95,10 @@ abstract public class Host {
                     throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
                 }
 
-                try (
-                        InputStream downloadStream = response.getEntity().getContent();
-                        FileOutputStream fos = new FileOutputStream(destinationFolder.getPath() + File.separator + sanitize(imageFileData.getImageName()))
-                ) {
-
+                File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
+                InputStream downloadStream = response.getEntity().getContent();
+                FileOutputStream fos = new FileOutputStream(outputFile);
+                try {
                     image.setTotal(response.getEntity().getContentLength());
                     logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
                     logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
@@ -110,7 +111,15 @@ abstract public class Host {
                         downloadSpeedService.increase(read);
                     }
                     EntityUtils.consumeQuietly(response.getEntity());
+                } finally {
+                    if (downloadStream != null) {
+                        downloadStream.close();
+                    }
+                    if (fos != null) {
+                        fos.close();
+                    }
                 }
+                checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
             }
         } catch (Exception e) {
             if(Thread.interrupted()) {
@@ -120,6 +129,46 @@ abstract public class Host {
         }
     }
 
+    private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
+        String formatName;
+        try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
+            Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
+            if (!it.hasNext()) {
+                throw new HostException("Image file is not recognized!");
+            }
+            ImageReader reader = it.next();
+            formatName = reader.getFormatName();
+            if (formatName.toUpperCase().equals("JPEG")) {
+                formatName = "jpg";
+            }
+        } catch (Exception e) {
+            throw new HostException("Failed to guess image format", e);
+        }
+        try {
+            String outImageName = (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase();
+            Files.move(outputFile.toPath(), new File(outputFile.getParent(), outImageName).toPath());
+        } catch (Exception e) {
+            throw new HostException("Failed to rename the image", e);
+        }
+    }
+
+    /**
+     * Will sanitize the image name and remove extension
+     *
+     * @param imageName
+     * @return
+     */
+    protected String formatImageFileName(String imageName) {
+        int extensionIndex = imageName.lastIndexOf('.');
+        String fileName;
+        if (extensionIndex != -1) {
+            fileName = imageName.substring(0, extensionIndex);
+        } else {
+            fileName = imageName;
+        }
+        return sanitize(fileName);
+    }
+
     /**
      * Just for testing, you may ignore
      * @throws Exception

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -34,6 +34,7 @@ public class AppSettingsService {
     private final String V_PASSWORD = "VPASSWORD";
     private final String V_THANKS = "VTHANKS";
     private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
+    private final String FORCE_ORDER = "FORCE_ORDER";
 
     private String downloadPath;
     private int maxThreads;
@@ -43,6 +44,7 @@ public class AppSettingsService {
     private String vPassword;
     private boolean vThanks;
     private boolean desktopClipboard;
+    private boolean forceOrder;
 
     public void setVPassword(String vPassword) {
         if(vPassword.isEmpty()) {
@@ -62,6 +64,7 @@ public class AppSettingsService {
         vPassword = prefs.get(V_PASSWORD, "");
         vThanks = prefs.getBoolean(V_THANKS, false);
         desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
+        forceOrder = prefs.getBoolean(FORCE_ORDER, false);
     }
 
     @PreDestroy
@@ -75,6 +78,7 @@ public class AppSettingsService {
         prefs.put(V_PASSWORD, vPassword);
         prefs.putBoolean(V_THANKS, vThanks);
         prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
+        prefs.putBoolean(FORCE_ORDER, forceOrder);
 
         try {
             prefs.sync();
@@ -121,8 +125,10 @@ public class AppSettingsService {
         private boolean vThanks;
         @JsonProperty("desktopClipboard")
         private boolean desktopClipboard;
+        @JsonProperty("forceOrder")
+        private boolean forceOrder;
 
-        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard) {
+        public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder) {
             this.downloadPath = downloadPath;
             this.maxThreads = maxThreads;
             this.autoStart = autoStart;
@@ -131,6 +137,7 @@ public class AppSettingsService {
             this.vPassword = vPassword;
             this.vThanks = vThanks;
             this.desktopClipboard = desktopClipboard;
+            this.forceOrder = forceOrder;
         }
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java

@@ -45,8 +45,8 @@ public class AppStateService {
         }
     }
 
-    public Image createImage(String pageUrl, String postId, String postName, Host host) {
-        return new Image(pageUrl, postId, postName, host, this);
+    public Image createImage(String pageUrl, String postId, String postName, Host host, int index) {
+        return new Image(pageUrl, postId, postName, host, this, index);
     }
 
     public Post createPost(String title, String url, List<Image> images, Map<String, String> metadata, String postId, String postCounter) {

vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java

@@ -191,7 +191,7 @@ public class PostParser {
                     }
                     if (foundHost != null) {
                         logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref));
-                        imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost));
+                        imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost, imagesList.size() + 1));
                     } else {
                         logger.warn(String.format("unsupported host for %s, skipping", imageHref));
                         continue;

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -7,8 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.exception.ValidationException;
+import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.VipergirlsAuthService;
 
 @RestController
@@ -57,6 +57,7 @@ public class SettingsRestEndpoint {
             this.settings.setVThanks(false);
         }
         this.settings.setDesktopClipboard(settings.isDesktopClipboard());
+        this.settings.setForceOrder(settings.isForceOrder());
 
         this.settings.save();
 
@@ -76,7 +77,8 @@ public class SettingsRestEndpoint {
                 settings.getVUsername(),
                 settings.getVPassword(),
                 settings.isVThanks(),
-                settings.isDesktopClipboard()
+                settings.isDesktopClipboard(),
+                settings.isForceOrder()
         );
     }
 

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "1.4.6",
+  "version": "1.4.10",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "1.4.6",
+  "version": "1.4.10",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>1.4.6</version>
+    <version>1.4.10</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/settings/settings.component.html

@@ -35,6 +35,11 @@
               />
             </mat-form-field>
 
+            <mat-checkbox color="primary" formControlName="forceOrder" name="forceOrder"
+            >Force image ordering (prepend incremental numbers)
+            </mat-checkbox
+            >
+
             <mat-checkbox color="primary" formControlName="autoStart" name="autoStart"
               >Auto start downloads</mat-checkbox
             >

vripper-ui/src/app/settings/settings.component.scss

@@ -18,3 +18,7 @@
 .container form section div > * {
   width: 100%;
 }
+
+mat-checkbox {
+  display: block;
+}

vripper-ui/src/app/settings/settings.component.ts

@@ -25,6 +25,7 @@ export class SettingsComponent implements OnInit {
     downloadPath: new FormControl(''),
     maxThreads: new FormControl(''),
     autoStart: new FormControl(false),
+    forceOrder: new FormControl(false),
     vLogin: new FormControl(false),
     vUsername: new FormControl(''),
     vPassword: new FormControl(''),