diff --git a/CHANGELOG.md b/CHANGELOG.md index 524f385..9e1101c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.4.10] - 2019-08-15 +### Changed +- Fix a rename issue on windows + +## [1.4.9] - 2019-08-13 +### Changed +- Update Built-By to be generic + +## [1.4.8] - 2019-08-13 +### Changed +- Add option to force image ordering + +## [1.4.7] - 2019-08-13 +### Changed +- Fixed a bug in image filename extensions + ## [1.4.6] - 2019-08-06 ### Changed - Fixed a bug when creating image file names diff --git a/pom.xml b/pom.xml index 01dac29..ae18769 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 tn.mnlr vripper - 1.4.6 + 1.4.10 pom org.springframework.boot diff --git a/vripper-electron/package-lock.json b/vripper-electron/package-lock.json index 8bb5c00..072c4fe 100644 --- a/vripper-electron/package-lock.json +++ b/vripper-electron/package-lock.json @@ -1,6 +1,6 @@ { "name": "vripper-electron", - "version": "1.4.6", + "version": "1.4.10", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/vripper-electron/package.json b/vripper-electron/package.json index d872239..ec5a769 100644 --- a/vripper-electron/package.json +++ b/vripper-electron/package.json @@ -1,6 +1,6 @@ { "name": "vripper-electron", - "version": "1.4.6", + "version": "1.4.10", "description": "", "main": "main.js", "author": "", diff --git a/vripper-electron/pom.xml b/vripper-electron/pom.xml index 7fb20b6..9a8e30c 100644 --- a/vripper-electron/pom.xml +++ b/vripper-electron/pom.xml @@ -5,7 +5,7 @@ tn.mnlr vripper - 1.4.6 + 1.4.10 vripper-electron vripper-electron diff --git a/vripper-server/pom.xml b/vripper-server/pom.xml index eca74c7..3077415 100644 --- a/vripper-server/pom.xml +++ b/vripper-server/pom.xml @@ -5,7 +5,7 @@ tn.mnlr vripper - 1.4.6 + 1.4.10 vripper-server vripper-server @@ -129,6 +129,7 @@ ${maven.build.timestamp} + ${user.name} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java b/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java index fd4ce0f..ba240b3 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java @@ -9,7 +9,6 @@ import tn.mnlr.vripper.host.Host; import tn.mnlr.vripper.services.AppStateService; import java.util.Objects; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @Getter @@ -30,6 +29,8 @@ public class Image { private String url; + private int index; + private AtomicLong current = new AtomicLong(0); private Status status; private BehaviorProcessor imageStateProcessor; @@ -37,12 +38,13 @@ public class Image { @Setter private long total = 0; - public Image(String url, String postId, String postName, Host host, AppStateService appStateService) { + public Image(String url, String postId, String postName, Host host, AppStateService appStateService, int index) { this.url = url; this.postId = postId; this.postName = postName; this.host = host; this.appStateService = appStateService; + this.index = index; status = Status.PENDING; imageStateProcessor = BehaviorProcessor.create(); appStateService.getCurrentImages().put(this.url, this); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java index 64ab567..d109c59 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java @@ -18,10 +18,15 @@ import tn.mnlr.vripper.exception.HtmlProcessorException; import tn.mnlr.vripper.q.ImageFileData; import tn.mnlr.vripper.services.*; +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.ImageInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.util.Iterator; import java.util.Random; @Service @@ -72,10 +77,7 @@ abstract public class Host { * END HOST SPECIFIC */ - if (!imageFileData.getImageName().toLowerCase().endsWith(".jpg") && !imageFileData.getImageName().toLowerCase().endsWith(".jpeg")) { - imageFileData.setImageName(imageFileData.getImageName() + ".jpg"); - } - + imageFileData.setImageName(formatImageFileName(imageFileData.getImageName())); File destinationFolder = new File(appSettingsService.getDownloadPath(), sanitize(image.getPostName() + "_" + image.getPostId())); logger.info(String.format("Saving to %s", destinationFolder.getPath())); if (!destinationFolder.exists()) { @@ -93,11 +95,10 @@ abstract public class Host { throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode())); } - try ( - InputStream downloadStream = response.getEntity().getContent(); - FileOutputStream fos = new FileOutputStream(destinationFolder.getPath() + File.separator + sanitize(imageFileData.getImageName())) - ) { - + File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp"); + InputStream downloadStream = response.getEntity().getContent(); + FileOutputStream fos = new FileOutputStream(outputFile); + try { image.setTotal(response.getEntity().getContentLength()); logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal())); logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl())); @@ -110,7 +111,15 @@ abstract public class Host { downloadSpeedService.increase(read); } EntityUtils.consumeQuietly(response.getEntity()); + } finally { + if (downloadStream != null) { + downloadStream.close(); + } + if (fos != null) { + fos.close(); + } } + checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex()); } } catch (Exception e) { if(Thread.interrupted()) { @@ -120,6 +129,46 @@ abstract public class Host { } } + private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException { + String formatName; + try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) { + Iterator it = ImageIO.getImageReaders(iis); + if (!it.hasNext()) { + throw new HostException("Image file is not recognized!"); + } + ImageReader reader = it.next(); + formatName = reader.getFormatName(); + if (formatName.toUpperCase().equals("JPEG")) { + formatName = "jpg"; + } + } catch (Exception e) { + throw new HostException("Failed to guess image format", e); + } + try { + String outImageName = (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase(); + Files.move(outputFile.toPath(), new File(outputFile.getParent(), outImageName).toPath()); + } catch (Exception e) { + throw new HostException("Failed to rename the image", e); + } + } + + /** + * Will sanitize the image name and remove extension + * + * @param imageName + * @return + */ + protected String formatImageFileName(String imageName) { + int extensionIndex = imageName.lastIndexOf('.'); + String fileName; + if (extensionIndex != -1) { + fileName = imageName.substring(0, extensionIndex); + } else { + fileName = imageName; + } + return sanitize(fileName); + } + /** * Just for testing, you may ignore * @throws Exception diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java index 4991421..6e372cd 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java @@ -34,6 +34,7 @@ public class AppSettingsService { private final String V_PASSWORD = "VPASSWORD"; private final String V_THANKS = "VTHANKS"; private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD"; + private final String FORCE_ORDER = "FORCE_ORDER"; private String downloadPath; private int maxThreads; @@ -43,6 +44,7 @@ public class AppSettingsService { private String vPassword; private boolean vThanks; private boolean desktopClipboard; + private boolean forceOrder; public void setVPassword(String vPassword) { if(vPassword.isEmpty()) { @@ -62,6 +64,7 @@ public class AppSettingsService { vPassword = prefs.get(V_PASSWORD, ""); vThanks = prefs.getBoolean(V_THANKS, false); desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false); + forceOrder = prefs.getBoolean(FORCE_ORDER, false); } @PreDestroy @@ -75,6 +78,7 @@ public class AppSettingsService { prefs.put(V_PASSWORD, vPassword); prefs.putBoolean(V_THANKS, vThanks); prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard); + prefs.putBoolean(FORCE_ORDER, forceOrder); try { prefs.sync(); @@ -121,8 +125,10 @@ public class AppSettingsService { private boolean vThanks; @JsonProperty("desktopClipboard") private boolean desktopClipboard; + @JsonProperty("forceOrder") + private boolean forceOrder; - public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard) { + public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder) { this.downloadPath = downloadPath; this.maxThreads = maxThreads; this.autoStart = autoStart; @@ -131,6 +137,7 @@ public class AppSettingsService { this.vPassword = vPassword; this.vThanks = vThanks; this.desktopClipboard = desktopClipboard; + this.forceOrder = forceOrder; } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java index 451170c..002be95 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java @@ -45,8 +45,8 @@ public class AppStateService { } } - public Image createImage(String pageUrl, String postId, String postName, Host host) { - return new Image(pageUrl, postId, postName, host, this); + public Image createImage(String pageUrl, String postId, String postName, Host host, int index) { + return new Image(pageUrl, postId, postName, host, this, index); } public Post createPost(String title, String url, List images, Map metadata, String postId, String postCounter) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java index 6723248..bfdfd6a 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java @@ -191,7 +191,7 @@ public class PostParser { } if (foundHost != null) { logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref)); - imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost)); + imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost, imagesList.size() + 1)); } else { logger.warn(String.format("unsupported host for %s, skipping", imageHref)); continue; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java index 2db2591..e486c6c 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java @@ -7,8 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import tn.mnlr.vripper.services.AppSettingsService; import tn.mnlr.vripper.exception.ValidationException; +import tn.mnlr.vripper.services.AppSettingsService; import tn.mnlr.vripper.services.VipergirlsAuthService; @RestController @@ -57,6 +57,7 @@ public class SettingsRestEndpoint { this.settings.setVThanks(false); } this.settings.setDesktopClipboard(settings.isDesktopClipboard()); + this.settings.setForceOrder(settings.isForceOrder()); this.settings.save(); @@ -76,7 +77,8 @@ public class SettingsRestEndpoint { settings.getVUsername(), settings.getVPassword(), settings.isVThanks(), - settings.isDesktopClipboard() + settings.isDesktopClipboard(), + settings.isForceOrder() ); } diff --git a/vripper-ui/package-lock.json b/vripper-ui/package-lock.json index d86a77a..74a3237 100644 --- a/vripper-ui/package-lock.json +++ b/vripper-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "vripper-ui", - "version": "1.4.6", + "version": "1.4.10", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/vripper-ui/package.json b/vripper-ui/package.json index f7df0e3..80f9fa5 100644 --- a/vripper-ui/package.json +++ b/vripper-ui/package.json @@ -1,6 +1,6 @@ { "name": "vripper-ui", - "version": "1.4.6", + "version": "1.4.10", "scripts": { "ng": "ng", "start": "ng serve", diff --git a/vripper-ui/pom.xml b/vripper-ui/pom.xml index d748cbc..a61b7f4 100644 --- a/vripper-ui/pom.xml +++ b/vripper-ui/pom.xml @@ -5,7 +5,7 @@ tn.mnlr vripper - 1.4.6 + 1.4.10 vripper-ui vripper-ui diff --git a/vripper-ui/src/app/settings/settings.component.html b/vripper-ui/src/app/settings/settings.component.html index 9342aee..c70343a 100644 --- a/vripper-ui/src/app/settings/settings.component.html +++ b/vripper-ui/src/app/settings/settings.component.html @@ -35,6 +35,11 @@ /> + Force image ordering (prepend incremental numbers) + + Auto start downloads diff --git a/vripper-ui/src/app/settings/settings.component.scss b/vripper-ui/src/app/settings/settings.component.scss index db5f0da..a804cbc 100644 --- a/vripper-ui/src/app/settings/settings.component.scss +++ b/vripper-ui/src/app/settings/settings.component.scss @@ -18,3 +18,7 @@ .container form section div > * { width: 100%; } + +mat-checkbox { + display: block; +} \ No newline at end of file diff --git a/vripper-ui/src/app/settings/settings.component.ts b/vripper-ui/src/app/settings/settings.component.ts index 34ae9f7..2306cda 100644 --- a/vripper-ui/src/app/settings/settings.component.ts +++ b/vripper-ui/src/app/settings/settings.component.ts @@ -25,6 +25,7 @@ export class SettingsComponent implements OnInit { downloadPath: new FormControl(''), maxThreads: new FormControl(''), autoStart: new FormControl(false), + forceOrder: new FormControl(false), vLogin: new FormControl(false), vUsername: new FormControl(''), vPassword: new FormControl(''),