v3.0.9

Fix a bug that lead to adding duplicate posts
Remove old unused settings

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [3.0.9] - 2020-09-27
+### Changed
+- Revert to appending post id to download folders
+- Fix a bug that lead to adding duplicate posts
+- Remove old unused settings
+ 
+## [3.0.8] - 2020-08-22
+### Changed
+- Bug fix which breaks concurrent downloads
+
 ## [3.0.7] - 2020-08-22
 ### Changed
 - Fix MacOs build

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.0.7</version>
+    <version>3.0.9</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.0.7</version>
+    <version>3.0.9</version>
   </parent>
   <artifactId>vripper-electron</artifactId>
   <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>3.0.7</version>
+        <version>3.0.9</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>

vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java

@@ -71,7 +71,7 @@ public class AcidimgHost extends Host {
         if (contDiv != null) {
             log.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
-            HttpPost httpPost = cm.buildHttpPost(url);
+            HttpPost httpPost = cm.buildHttpPost(url, context);
             httpPost.addHeader("Referer", url);
             List<NameValuePair> params = new ArrayList<>();
             params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));

vripper-server/src/main/java/tn/mnlr/vripper/host/ImageBamHost.java

@@ -68,7 +68,7 @@ public class ImageBamHost extends Host {
         if (contDiv != null) {
             log.debug(String.format("Click button found for %s", url));
             HttpClient client = cm.getClient().build();
-            HttpGet httpGet = cm.buildHttpGet(url);
+            HttpGet httpGet = cm.buildHttpGet(url, context);
             httpGet.addHeader("Referer", url);
             log.debug(String.format("Requesting %s", httpGet));
             try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -81,7 +81,7 @@ public class ImxHost extends Host {
         }
         log.debug(String.format("Click button found for %s", url));
         HttpClient client = cm.getClient().build();
-        HttpPost httpPost = cm.buildHttpPost(url);
+        HttpPost httpPost = cm.buildHttpPost(url, context);
         List<NameValuePair> params = new ArrayList<>();
         params.add(new BasicNameValuePair("imgContinue", value));
         try {

vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Post.java

@@ -52,8 +52,6 @@ public class Post {
 
     private Metadata metadata;
 
-    private boolean renaming;
-
     public Post(String title, String url, String postId, String threadId, String threadTitle, String forum, String securityToken) {
         this.title = title;
         this.url = url;

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRepository.java

@@ -78,7 +78,7 @@ public class MetadataRepository implements IMetadataRepository {
     @Override
     public int deleteByPostId(String postId) {
         return jdbcTemplate.update(
-                "DELETE FROM METADATA WHERE POST_ID_REF = (SELECT post.ID FROM POST AS post INNER JOIN METADATA metadata ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?)",
+                "DELETE FROM METADATA AS metadata WHERE metadata.ID = (SELECT inner_metadata.ID FROM POST AS post INNER JOIN METADATA inner_metadata ON post.ID = inner_metadata.POST_ID_REF WHERE post.POST_ID = ?)",
                 postId
         );
     }

vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java

@@ -4,6 +4,7 @@ import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.failsafe.function.CheckedRunnable;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.AbstractExecutionAwareRequest;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.protocol.HttpClientContext;
@@ -25,13 +26,28 @@ import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
-import java.util.Iterator;
-import java.util.Objects;
+import java.util.*;
+import java.util.concurrent.locks.ReentrantLock;
 
 @Slf4j
 public class DownloadJob implements CheckedRunnable {
 
-    private static final Byte LOCK = 0;
+    public enum ContextAttributes {
+        OPEN_CONNECTION("OPEN_CONNECTION");
+
+        private final String value;
+
+        ContextAttributes(String value) {
+            this.value = value;
+        }
+
+        @Override
+        public String toString() {
+            return value;
+        }
+    }
+
+    private static final ReentrantLock LOCK = new ReentrantLock();
     private static final int READ_BUFFER_SIZE = 8192;
 
     private final DataService dataService;
@@ -41,15 +57,14 @@ public class DownloadJob implements CheckedRunnable {
     private final DownloadSpeedService downloadSpeedService;
     private final AppSettingsService appSettingsService;
 
+    private final HttpClientContext context;
+
     @Getter
     private final Image image;
 
     @Getter
     private final Post post;
 
-    @Getter
-    private final ImageFileData imageFileData = new ImageFileData();
-
     private boolean stopped = false;
 
     @Getter
@@ -64,9 +79,12 @@ public class DownloadJob implements CheckedRunnable {
         authService = SpringContext.getBean(VipergirlsAuthService.class);
         downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
         appSettingsService = SpringContext.getBean(AppSettingsService.class);
+        context = HttpClientContext.create();
+        context.setCookieStore(new BasicCookieStore());
+        context.setAttribute(ContextAttributes.OPEN_CONNECTION.toString(), Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
     }
 
-    public void download(final Post post, final Image image, final ImageFileData imageFileData) throws DownloadException {
+    public void download(final Post post, final Image image) throws DownloadException {
 
         try {
 
@@ -75,18 +93,17 @@ public class DownloadJob implements CheckedRunnable {
             dataService.updateImageStatus(image.getStatus(), image.getId());
             dataService.updateImageCurrent(image.getCurrent(), image.getId());
 
-            HttpClientContext context = HttpClientContext.create();
-            context.setCookieStore(new BasicCookieStore());
-
-            synchronized (LOCK) {
+            try {
+                LOCK.lock();
                 if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
                     post.setStatus(Status.DOWNLOADING);
                     dataService.updatePostStatus(post.getStatus(), post.getId());
                 }
+            } finally {
+                LOCK.unlock();
             }
 
 
-            imageFileData.setPageUrl(image.getUrl());
             if (stopped) {
                 return;
             }
@@ -111,8 +128,8 @@ public class DownloadJob implements CheckedRunnable {
             HttpClient client = cm.getClient().build();
 
             log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
-            HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl());
-            httpGet.addHeader("Referer", imageFileData.getPageUrl());
+            HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
+            httpGet.addHeader("Referer", image.getUrl());
             try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
 
                 if (response.getStatusLine().getStatusCode() / 100 != 2) {
@@ -123,13 +140,18 @@ public class DownloadJob implements CheckedRunnable {
                     return;
                 }
                 File destinationFolder;
-                synchronized (LOCK) {
+                try {
+                    LOCK.lock();
                     Post updatedPost = dataService.findPostById(post.getId()).orElseThrow();
                     if (updatedPost.getPostFolderName() == null) {
                         pathService.createDefaultPostFolder(updatedPost);
                     }
                     destinationFolder = pathService.getDownloadDestinationFolder(updatedPost);
-                    authService.leaveThanks(updatedPost);
+                    if (appSettingsService.getSettings().getLeaveThanksOnStart()) {
+                        authService.leaveThanks(updatedPost);
+                    }
+                } finally {
+                    LOCK.unlock();
                 }
                 File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + nameAndUrl.getName() + ".tmp");
                 try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
@@ -158,10 +180,12 @@ public class DownloadJob implements CheckedRunnable {
                         return;
                     }
                 }
-                File finalName = checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
-                imageFileData.setFileName(finalName.getName());
+                checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
             }
         } catch (Exception e) {
+            if (stopped) {
+                return;
+            }
             throw new DownloadException(e);
         } finally {
             if (image.getCurrent() == image.getTotal()) {
@@ -176,7 +200,7 @@ public class DownloadJob implements CheckedRunnable {
         }
     }
 
-    private File checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
+    private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
         try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
             Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
             if (!it.hasNext()) {
@@ -221,7 +245,7 @@ public class DownloadJob implements CheckedRunnable {
             if (outImage.exists() && outImage.delete()) {
                 log.debug(String.format("%s is deleted", outImage.toString()));
             }
-            return Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
+            Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
         } catch (Exception e) {
             throw new HostException("Failed to rename the image", e);
         }
@@ -234,7 +258,7 @@ public class DownloadJob implements CheckedRunnable {
             return;
         }
         log.debug(String.format("Starting downloading %s", image.getUrl()));
-        download(post, image, imageFileData);
+        download(post, image);
     }
 
     @Override
@@ -252,6 +276,12 @@ public class DownloadJob implements CheckedRunnable {
     }
 
     public void stop() {
+        List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
+        if (requests != null) {
+            for (AbstractExecutionAwareRequest request : requests) {
+                request.abort();
+            }
+        }
         this.stopped = true;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecuteRunnable.java

@@ -6,31 +6,23 @@ import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.VripperApplication;
 import tn.mnlr.vripper.jpa.domain.enums.Status;
 import tn.mnlr.vripper.services.DataService;
-import tn.mnlr.vripper.services.MutexService;
-
-import java.util.concurrent.locks.ReentrantLock;
 
 @Slf4j
 public class ExecuteRunnable implements Runnable {
 
     private final ExecutionService executionService;
     private final DataService dataService;
-    private final MutexService mutexService;
 
     private final DownloadJob downloadJob;
 
     public ExecuteRunnable(final DownloadJob downloadJob) {
         executionService = SpringContext.getBean(ExecutionService.class);
         dataService = SpringContext.getBean(DataService.class);
-        mutexService = SpringContext.getBean(MutexService.class);
         this.downloadJob = downloadJob;
     }
 
     @Override
     public void run() {
-        mutexService.createPostLock(downloadJob.getPost().getPostId());
-        ReentrantLock mutex = mutexService.getPostLock(downloadJob.getPost().getPostId());
-        mutex.lock();
         Failsafe.with(VripperApplication.retryPolicy)
                 .onFailure(e -> {
                     log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
@@ -41,7 +33,6 @@ public class ExecuteRunnable implements Runnable {
                     dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
                     executionService.afterJobFinish(downloadJob);
                     log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
-                    mutex.unlock();
                 }).run(downloadJob);
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java

@@ -10,7 +10,6 @@ import tn.mnlr.vripper.jpa.domain.Post;
 import tn.mnlr.vripper.jpa.domain.enums.Status;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.DataService;
-import tn.mnlr.vripper.services.MutexService;
 import tn.mnlr.vripper.services.post.PostService;
 
 import javax.annotation.PostConstruct;
@@ -35,19 +34,17 @@ public class ExecutionService {
     private final AppSettingsService settings;
     private final DataService dataService;
     private final PostService postService;
-    private final MutexService mutexService;
 
     private boolean pauseQ = false;
     private Thread executionThread;
     private Thread pollThread;
 
     @Autowired
-    public ExecutionService(PendingQ pendingQ, AppSettingsService settings, DataService dataService, PostService postService, MutexService mutexService) {
+    public ExecutionService(PendingQ pendingQ, AppSettingsService settings, DataService dataService, PostService postService) {
         this.pendingQ = pendingQ;
         this.settings = settings;
         this.dataService = dataService;
         this.postService = postService;
-        this.mutexService = mutexService;
     }
 
     @PostConstruct
@@ -214,7 +211,6 @@ public class ExecutionService {
         int count = pendingQ.decrement(downloadJob.getPost().getPostId());
         if (count == 0) {
             dataService.finishPost(downloadJob.getPost());
-            mutexService.removePostLock(downloadJob.getPost().getPostId());
         }
         threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
         executing.remove(downloadJob);

vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java

@@ -1,12 +0,0 @@
-package tn.mnlr.vripper.q;
-
-import lombok.Getter;
-import lombok.Setter;
-
-@Getter
-@Setter
-public class ImageFileData {
-
-    private String pageUrl;
-    private String fileName;
-}

vripper-server/src/main/java/tn/mnlr/vripper/services/AppSettingsService.java

@@ -30,16 +30,12 @@ import static java.nio.file.StandardOpenOption.*;
 @Slf4j
 public class AppSettingsService {
 
-    private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
-    private final String baseDir;
-
     private final Path configPath;
     private final ObjectMapper om = new ObjectMapper();
 
     private Settings settings = new Settings();
 
     public AppSettingsService(@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
-        this.baseDir = baseDir;
         this.configPath = Paths.get(baseDir, baseDirName, "config.json");
         om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
     }
@@ -53,7 +49,7 @@ public class AppSettingsService {
 
         if (settings.getVLogin()) {
             if (!this.settings.getVPassword().equals(settings.getVPassword())) {
-                settings.setVPassword(settings.getVPassword());
+                settings.setVPassword(DigestUtils.md5Hex(settings.getVPassword()));
             }
         } else {
             settings.setVUsername("");
@@ -135,8 +131,12 @@ public class AppSettingsService {
             settings.setDarkTheme(false);
         }
 
-        if (settings.getViewPhotos() == null) {
-            settings.setViewPhotos(false);
+        if (settings.getAppendPostId() == null) {
+            settings.setAppendPostId(true);
+        }
+
+        if (settings.getLeaveThanksOnStart() == null) {
+            settings.setLeaveThanksOnStart(false);
         }
 
         save();
@@ -145,8 +145,6 @@ public class AppSettingsService {
     @PreDestroy
     public void save() {
         try {
-            // force disable gallery
-            settings.setViewPhotos(false);
 
             Files.write(configPath, om.writeValueAsBytes(settings), CREATE, WRITE, TRUNCATE_EXISTING, SYNC);
         } catch (IOException e) {
@@ -206,41 +204,50 @@ public class AppSettingsService {
 
         @JsonProperty("downloadPath")
         private String downloadPath;
+
         @JsonProperty("maxThreads")
         private Integer maxThreads;
+
         @JsonProperty("maxTotalThreads")
         private Integer maxTotalThreads;
+
         @JsonProperty("autoStart")
         private Boolean autoStart;
+
         @JsonProperty("vLogin")
         private Boolean vLogin;
+
         @JsonProperty("vUsername")
         private String vUsername;
+
         @JsonProperty("vPassword")
         private String vPassword;
+
         @JsonProperty("vThanks")
         private Boolean vThanks;
+
         @JsonProperty("desktopClipboard")
         private Boolean desktopClipboard;
+
         @JsonProperty("forceOrder")
         private Boolean forceOrder;
+
         @JsonProperty("subLocation")
         private Boolean subLocation;
+
         @JsonProperty("threadSubLocation")
         private Boolean threadSubLocation;
+
         @JsonProperty("clearCompleted")
         private Boolean clearCompleted;
-        @JsonProperty("viewPhotos")
-        private Boolean viewPhotos;
+
         @JsonProperty("darkTheme")
         private Boolean darkTheme;
 
-        public void setVPassword(String vPassword) {
-            if (vPassword == null || vPassword.isEmpty()) {
-                this.vPassword = "";
-            } else {
-                this.vPassword = DigestUtils.md5Hex(vPassword);
-            }
-        }
+        @JsonProperty("appendPostId")
+        private Boolean appendPostId;
+
+        @JsonProperty("leaveThanksOnStart")
+        private Boolean leaveThanksOnStart;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java

@@ -2,8 +2,10 @@ package tn.mnlr.vripper.services;
 
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.AbstractExecutionAwareRequest;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.client.LaxRedirectStrategy;
@@ -11,8 +13,10 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.q.DownloadJob;
 
 import java.net.URI;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 @Service
@@ -53,21 +57,33 @@ public class ConnectionManager {
                 .setDefaultRequestConfig(rc);
     }
 
-    public HttpGet buildHttpGet(String url) {
+    public HttpGet buildHttpGet(String url, final HttpClientContext context) {
         HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
         httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
+        addToContext(context, httpGet);
         return httpGet;
     }
 
-    public HttpPost buildHttpPost(String url) {
+    public HttpPost buildHttpPost(String url, final HttpClientContext context) {
         HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
         httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
+        addToContext(context, httpPost);
         return httpPost;
     }
 
-    public HttpGet buildHttpGet(URI uri) {
+    public HttpGet buildHttpGet(URI uri, final HttpClientContext context) {
         HttpGet httpGet = new HttpGet(uri);
         httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
+        addToContext(context, httpGet);
         return httpGet;
     }
+
+    public void addToContext(HttpClientContext context, AbstractExecutionAwareRequest request) {
+        if (context != null) {
+            List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
+            if (requests != null) {
+                requests.add(request);
+            }
+        }
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java

@@ -253,8 +253,4 @@ public class DataService {
         postRepository.updateThanked(thanked, id);
         livePostsState.onNext(id);
     }
-
-    public void refreshPost(Long id) {
-        livePostsState.onNext(id);
-    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/HostService.java

@@ -60,7 +60,7 @@ public class HostService {
         String basePage;
 
         HttpClient client = cm.getClient().build();
-        HttpGet httpGet = cm.buildHttpGet(url);
+        HttpGet httpGet = cm.buildHttpGet(url, context);
         Header[] headers;
         log.debug(String.format("Requesting %s", url));
         try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {

vripper-server/src/main/java/tn/mnlr/vripper/services/MutexService.java

@@ -1,27 +0,0 @@
-package tn.mnlr.vripper.services;
-
-import org.springframework.stereotype.Service;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
-
-@Service
-public class MutexService {
-
-    private final Map<String, ReentrantLock> postLock = new ConcurrentHashMap<>();
-
-    public synchronized void createPostLock(String postId) {
-        if (!postLock.containsKey(postId)) {
-            postLock.put(postId, new ReentrantLock());
-        }
-    }
-
-    public void removePostLock(String postId) {
-        postLock.remove(postId);
-    }
-
-    public ReentrantLock getPostLock(String postId) {
-        return postLock.get(postId);
-    }
-}

vripper-server/src/main/java/tn/mnlr/vripper/services/PathService.java

@@ -1,6 +1,5 @@
 package tn.mnlr.vripper.services;
 
-import lombok.Getter;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,99 +11,94 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
-import java.util.*;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 @Service
 @Slf4j
 public class PathService {
 
+    public static final int MAX_ATTEMPTS = 24;
     private final AppSettingsService appSettingsService;
     private final DataService dataService;
-    private final MutexService mutexService;
 
     private final CommonExecutor commonExecutor;
 
-    @Getter
-    private final Set<String> renaming = Collections.synchronizedSet(new HashSet<>());
-
     @Autowired
-    public PathService(AppSettingsService appSettingsService, DataService dataService, MutexService mutexService, CommonExecutor commonExecutor) {
+    public PathService(AppSettingsService appSettingsService, DataService dataService, CommonExecutor commonExecutor) {
         this.appSettingsService = appSettingsService;
         this.dataService = dataService;
-        this.mutexService = mutexService;
         this.commonExecutor = commonExecutor;
     }
 
     public final File getDownloadDestinationFolder(Post post) {
-        return _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), post.getPostFolderName());
+        return _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), post.getPostFolderName(), post.getPostId());
     }
 
-    private File _getDownloadDestinationFolder(@NonNull String forum, @NonNull String threadTitle, @NonNull String title) {
+    private File _getRootFolder(@NonNull String forum, @NonNull String threadTitle) {
         File sourceFolder = appSettingsService.getSettings().getSubLocation() ? new File(appSettingsService.getSettings().getDownloadPath(), sanitize(forum)) : new File(appSettingsService.getSettings().getDownloadPath());
-        sourceFolder = appSettingsService.getSettings().getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
+        return appSettingsService.getSettings().getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
+    }
+
+    private File _getDownloadDestinationFolder(@NonNull String forum, @NonNull String threadTitle, @NonNull String title, @NonNull String postId) {
+        File sourceFolder = _getRootFolder(forum, threadTitle);
         return new File(sourceFolder, title);
     }
 
+    private File _createDownloadDestinationFolder(@NonNull String forum, @NonNull String threadTitle, @NonNull String title, @NonNull String postId) {
+        File sourceFolder = _getRootFolder(forum, threadTitle);
+        return new File(sourceFolder, appSettingsService.getSettings().getAppendPostId() ? title + "_" + postId : title);
+    }
+
     public final void createDefaultPostFolder(Post post) {
-        File sourceFolder = _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(post.getTitle()));
+        File sourceFolder = _createDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(post.getTitle()), post.getPostId());
         File destFolder = makeDirs(sourceFolder);
         post.setPostFolderName(destFolder.getName());
         dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
     }
 
     public final void rename(@NonNull String postId, @NonNull String altName) {
-        renaming.add(postId);
         Post post = dataService.findPostByPostId(postId).orElseThrow();
-        dataService.refreshPost(post.getId());
         commonExecutor.getGeneralExecutor().submit(() -> {
-            ReentrantLock postLock = null;
-            try {
-                postLock = mutexService.getPostLock(postId);
-                if (postLock != null) {
-                    postLock.lock();
-                }
-                if (altName.equals(post.getTitle())) {
+            if (altName.equals(post.getTitle())) {
+                return;
+            }
+            post.setTitle(altName);
+            dataService.updatePostTitle(post.getTitle(), post.getId());
+            if (post.getPostFolderName() == null) {
+                return;
+            }
+            File newDestFolder = makeDirs(_getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(altName), postId));
+            File currentDesFolder = getDownloadDestinationFolder(post);
+            post.setPostFolderName(newDestFolder.getName());
+            dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
+
+            List<File> files = Arrays.stream(Objects.requireNonNull(currentDesFolder.listFiles())).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
+            for (File f : files) {
+                try {
+                    Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+                } catch (IOException e) {
+                    log.error(String.format("Failed to move files from %s to %s", currentDesFolder.toString(), newDestFolder.toString()), e);
                     return;
                 }
-                post.setTitle(altName);
-                dataService.updatePostTitle(post.getTitle(), post.getId());
-                if (post.getPostFolderName() == null) {
-                    return;
-                }
-                File newDestFolder = makeDirs(_getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(altName)));
-                File currentDesFolder = getDownloadDestinationFolder(post);
-                post.setPostFolderName(newDestFolder.getName());
-                dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
-
-                List<File> files = Arrays.stream(Objects.requireNonNull(currentDesFolder.listFiles())).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
-                for (File f : files) {
-                    try {
-                        Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
-                    } catch (IOException e) {
-                        log.error(String.format("Failed to move files from %s to %s", currentDesFolder.toString(), newDestFolder.toString()), e);
-                        return;
-                    }
-                }
-
-                for (File file : Objects.requireNonNull(currentDesFolder.listFiles())) {
-                    if (!file.delete()) {
-                        log.warn(String.format("Failed to remove %s", file.toString()));
-                    }
-                }
+            }
 
+            int attempts = 0;
+            while (currentDesFolder.exists() && attempts <= MAX_ATTEMPTS) {
+                attempts++;
                 if (!currentDesFolder.delete()) {
                     log.warn(String.format("Failed to remove %s", currentDesFolder.toString()));
                 }
-            } finally {
-                renaming.remove(postId);
-                dataService.refreshPost(post.getId());
-
-                if (postLock != null) {
-                    postLock.unlock();
+                try {
+                    Thread.sleep(5_000);
+                } catch (InterruptedException ignored) {
                 }
             }
+            if (attempts > MAX_ATTEMPTS) {
+                log.error(String.format("Failed to rename post %s", postId));
+            }
         });
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/services/VRThreadParser.java

@@ -49,7 +49,7 @@ public class VRThreadParser {
         try {
             URIBuilder uriBuilder = new URIBuilder(VR_API);
             uriBuilder.setParameter("t", queued.getThreadId());
-            httpGet = cm.buildHttpGet(uriBuilder.build());
+            httpGet = cm.buildHttpGet(uriBuilder.build(), null);
         } catch (URISyntaxException e) {
             throw new PostParseException(e);
         }

vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java

@@ -85,7 +85,7 @@ public class VipergirlsAuthService {
             return;
         }
 
-        HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login");
+        HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login", null);
         List<NameValuePair> params = new ArrayList<>();
         params.add(new BasicNameValuePair("vb_login_username", username));
 
@@ -161,7 +161,7 @@ public class VipergirlsAuthService {
 
     private void postThanks(Post post) throws VripperException {
 
-        HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php");
+        HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php", null);
         List<NameValuePair> params = new ArrayList<>();
         params.add(new BasicNameValuePair("do", "post_thanks_add"));
         params.add(new BasicNameValuePair("using_ajax", "1"));

vripper-server/src/main/java/tn/mnlr/vripper/services/post/MetadataCache.java

@@ -99,7 +99,7 @@ public class MetadataCache {
     }
 
     private Metadata fetchMetadata(Key key) {
-        HttpGet httpGet = cm.buildHttpGet(key.getUrl());
+        HttpGet httpGet = cm.buildHttpGet(key.getUrl(), null);
         Metadata metadata = new Metadata();
         Failsafe.with(VripperApplication.retryPolicy)
                 .onFailure(e -> {

vripper-server/src/main/java/tn/mnlr/vripper/services/post/PostService.java

@@ -11,6 +11,7 @@ import tn.mnlr.vripper.q.PendingQ;
 import tn.mnlr.vripper.services.AppSettingsService;
 import tn.mnlr.vripper.services.CommonExecutor;
 import tn.mnlr.vripper.services.DataService;
+import tn.mnlr.vripper.services.VipergirlsAuthService;
 
 import java.util.Map;
 import java.util.Set;
@@ -26,13 +27,15 @@ public class PostService {
     private final DataService dataService;
     private final CommonExecutor commonExecutor;
     private final Map<String, Future<?>> fetchingMetadata = new ConcurrentHashMap<>();
+    private final VipergirlsAuthService vipergirlsAuthService;
 
     @Autowired
-    public PostService(AppSettingsService appSettingsService, PendingQ pendingQ, DataService dataService, DataService dataService1, CommonExecutor commonExecutor) {
+    public PostService(AppSettingsService appSettingsService, PendingQ pendingQ, DataService dataService, CommonExecutor commonExecutor, VipergirlsAuthService vipergirlsAuthService) {
         this.appSettingsService = appSettingsService;
         this.pendingQ = pendingQ;
-        this.dataService = dataService1;
+        this.dataService = dataService;
         this.commonExecutor = commonExecutor;
+        this.vipergirlsAuthService = vipergirlsAuthService;
     }
 
     public void addPost(String postId, String threadId) throws PostParseException {
@@ -71,6 +74,9 @@ public class PostService {
             post.setStatus(Status.STOPPED);
             log.debug("Auto start downloads option is disabled");
         }
+        if (!appSettingsService.getSettings().getLeaveThanksOnStart()) {
+            vipergirlsAuthService.leaveThanks(post);
+        }
         dataService.updatePostStatus(post.getStatus(), post.getId());
     }
 

vripper-server/src/main/java/tn/mnlr/vripper/services/post/VRPostParser.java

@@ -44,7 +44,7 @@ public class VRPostParser {
         try {
             URIBuilder uriBuilder = new URIBuilder(VR_API);
             uriBuilder.setParameter("p", postId);
-            httpGet = cm.buildHttpGet(uriBuilder.build());
+            httpGet = cm.buildHttpGet(uriBuilder.build(), null);
         } catch (URISyntaxException e) {
             throw new PostParseException(e);
         }

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/AppDataController.java

@@ -60,7 +60,7 @@ public class AppDataController {
 
     @SubscribeMapping("/posts")
     public Collection<Post> posts() {
-        return StreamSupport.stream(dataService.findAllPosts().spliterator(), false).peek(this::isRenaming).collect(Collectors.toList());
+        return StreamSupport.stream(dataService.findAllPosts().spliterator(), false).collect(Collectors.toList());
     }
 
     @SubscribeMapping("/images/{postId}")
@@ -72,10 +72,4 @@ public class AppDataController {
     public Collection<Queued> queued() {
         return StreamSupport.stream(dataService.findAllQueued().spliterator(), false).collect(Collectors.toList());
     }
-
-    private void isRenaming(Post post) {
-        if (pathService.getRenaming().contains(post.getPostId())) {
-            post.setRenaming(true);
-        }
-    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketBroadcast.java

@@ -7,7 +7,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.messaging.simp.SimpMessagingTemplate;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.jpa.domain.Image;
-import tn.mnlr.vripper.jpa.domain.Post;
 import tn.mnlr.vripper.services.*;
 
 import javax.annotation.PostConstruct;
@@ -66,7 +65,7 @@ public class WebSocketBroadcast {
                 .buffer(500, TimeUnit.MILLISECONDS)
                 .map(HashSet::new)
                 .filter(e -> !e.isEmpty())
-                .subscribe(ids -> template.convertAndSend("/topic/posts", ids.stream().map(dataService::findPostById).filter(Optional::isPresent).map(Optional::get).peek(this::isRenaming).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e)));
+                .subscribe(ids -> template.convertAndSend("/topic/posts", ids.stream().map(dataService::findPostById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e)));
 
         disposables.add(dataService.liveImage()
                 .subscribeOn(Schedulers.io())
@@ -108,12 +107,6 @@ public class WebSocketBroadcast {
         );
     }
 
-    private void isRenaming(Post post) {
-        if (pathService.getRenaming().contains(post.getPostId())) {
-            post.setRenaming(true);
-        }
-    }
-
     @PreDestroy
     private void destroy() {
         disposables.forEach(Disposable::dispose);

vripper-server/src/main/resources/db/changelog/db.changelog-master.xml

@@ -164,4 +164,9 @@
                                  initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
                                  referencedColumnNames="ID" referencedTableName="POST"/>
     </changeSet>
+    <changeSet id="1600798657000-01" author="sysgen">
+        <createIndex tableName="POST" indexName="POST_UQ_POST_ID_IDX">
+            <column name="POST_ID"/>
+        </createIndex>
+    </changeSet>
 </databaseChangeLog>

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.0.7</version>
+    <version>3.0.9</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/domain/post-state.model.ts

@@ -11,8 +11,7 @@ export class PostState {
     public hosts: string[],
     public thanked: boolean,
     public previews: string[],
-    public metadata: Metadata,
-    public renaming: boolean
+    public metadata: Metadata
   ) {
   }
 }

vripper-ui/src/app/posts/renderer/post-progress.renderer.component.html

@@ -24,7 +24,7 @@
             <mat-icon class="icon" fxFlex="none" color="primary" [appPreview]="postState.previews">image</mat-icon>
             <span class="title">
               <p [title]="postState.title">{{
-                postState.renaming ? 'Renaming gallery...' : postState.title
+                postState.title
                 }}</p>
               <p><label>Alternative titles: </label><span
                 [title]="postState.metadata?.resolvedNames?.join(', ')">{{postState.metadata?.resolvedNames?.join(', ') || 'none'}}</span></p>

vripper-ui/src/app/ws-connection.service.ts

@@ -139,8 +139,7 @@ export class WsConnectionService {
               element.hosts,
               element.thanked,
               element.previews,
-              element.metadata,
-              element.renaming
+              element.metadata
             )
           );
         });

vripper-ui/src/environments/environment.prod.ts

@@ -2,5 +2,5 @@ export const environment = {
   production: true,
   localhost: `${window.location.protocol}//${window.location.host}`,
   ws: `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}`,
-  version: '3.0.7'
+  version: '3.0.9'
 };

vripper-ui/src/environments/environment.ts

@@ -6,7 +6,7 @@ export const environment = {
   production: false,
   localhost: 'http://localhost:8080',
   ws: 'ws://localhost:8080',
-  version: '3.0.7'
+  version: '3.0.9'
 };
 
 /*