diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/AppSettings.java b/vripper-server/src/main/java/tn/mnlr/vripper/AppSettings.java index 0632c1a..f767af6 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/AppSettings.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/AppSettings.java @@ -21,7 +21,7 @@ import java.util.prefs.Preferences; @Setter public class AppSettings { - private Logger logger = LoggerFactory.getLogger(AppSettings.class); + private static final Logger logger = LoggerFactory.getLogger(AppSettings.class); private Preferences prefs = Preferences.userNodeForPackage(tn.mnlr.vripper.AppSettings.class); @@ -51,24 +51,24 @@ public class AppSettings { public void restore() { - this.downloadPath = prefs.get(DOWNLOAD_PATH, ""); - this.maxThreads = prefs.getInt(MAX_THREADS, 1); - this.autoStart = prefs.getBoolean(AUTO_START, false); - this.vLogin = prefs.getBoolean(V_LOGIN, false); - this.vUsername = prefs.get(V_USERNAME, ""); - this.vPassword = prefs.get(V_PASSWORD, ""); - this.vThanks = prefs.getBoolean(V_THANKS, false); + downloadPath = prefs.get(DOWNLOAD_PATH, ""); + maxThreads = prefs.getInt(MAX_THREADS, 1); + autoStart = prefs.getBoolean(AUTO_START, false); + vLogin = prefs.getBoolean(V_LOGIN, false); + vUsername = prefs.get(V_USERNAME, ""); + vPassword = prefs.get(V_PASSWORD, ""); + vThanks = prefs.getBoolean(V_THANKS, false); } public void save() { - prefs.put(DOWNLOAD_PATH, this.downloadPath); - prefs.putInt(MAX_THREADS, this.maxThreads); - prefs.putBoolean(AUTO_START, this.autoStart); - prefs.putBoolean(V_LOGIN, this.vLogin); - prefs.put(V_USERNAME, this.vUsername); - prefs.put(V_PASSWORD, this.vPassword); - prefs.putBoolean(V_THANKS, this.vThanks); + prefs.put(DOWNLOAD_PATH, downloadPath); + prefs.putInt(MAX_THREADS, maxThreads); + prefs.putBoolean(AUTO_START, autoStart); + prefs.putBoolean(V_LOGIN, vLogin); + prefs.put(V_USERNAME, vUsername); + prefs.put(V_PASSWORD, vPassword); + prefs.putBoolean(V_THANKS, vThanks); try { prefs.sync(); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/Main.java b/vripper-server/src/main/java/tn/mnlr/vripper/Main.java deleted file mode 100644 index 937bb68..0000000 --- a/vripper-server/src/main/java/tn/mnlr/vripper/Main.java +++ /dev/null @@ -1,47 +0,0 @@ -package tn.mnlr.vripper; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; -import org.springframework.stereotype.Component; -import tn.mnlr.vripper.exception.VripperException; -import tn.mnlr.vripper.services.AppStateService; -import tn.mnlr.vripper.services.PersistenceService; -import tn.mnlr.vripper.services.VipergirlsAuthService; - -import java.io.File; - -@Component -public class Main implements ApplicationRunner { - - private Logger logger = LoggerFactory.getLogger(Main.class); - - public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json"; - - @Autowired - private VipergirlsAuthService authService; - - @Autowired - private PersistenceService persistenceService; - - @Autowired - private AppStateService stateService; - - @Autowired - private AppSettings appSettings; - - @Override - public void run(ApplicationArguments args) { - - persistenceService.restore(); - appSettings.restore(); - - try { - authService.authenticate(); - } catch (VripperException e) { - logger.error("Cannot authenticate user with ViperGirls", e); - } - } -} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java b/vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java index 2159f13..e4a8c82 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java @@ -1,18 +1,23 @@ package tn.mnlr.vripper; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.stereotype.Component; -import tn.mnlr.vripper.host.ImageZillaHost; -import tn.mnlr.vripper.q.DownloadQ; -import tn.mnlr.vripper.services.AppStateService; +import tn.mnlr.vripper.exception.VripperException; +import tn.mnlr.vripper.services.PersistenceService; +import tn.mnlr.vripper.services.VipergirlsAuthService; + +import java.io.File; @SpringBootApplication public class VripperApplication { + public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json"; + public static void main(String[] args) { SpringApplication.run(VripperApplication.class, args); } @@ -20,48 +25,27 @@ public class VripperApplication { @Component public class AppCommandRunner implements CommandLineRunner { - @Autowired - private AppStateService appStateService; + private final Logger logger = LoggerFactory.getLogger(AppCommandRunner.class); @Autowired - private ImageZillaHost imageZillaHost; + private VipergirlsAuthService authService; @Autowired - private DownloadQ downloadQ; + private PersistenceService persistenceService; - private ObjectMapper om = new ObjectMapper(); + @Autowired + private AppSettings appSettings; @Override - public void run(String... args) throws Exception { -// om.addMixIn(Image.class, ImageUIMixin.class).addMixIn(Post.class, PostUIMixin.class); -// -// Disposable subscription = FlowableProcessor.merge(appStateService.getAllImageState(), appStateService.getLiveImageUpdates()) -// .observeOn(Schedulers.io()) -// .filter(e -> e.getPostId().equals("001")) -// .buffer(2, TimeUnit.SECONDS) -// .filter(e -> !e.isEmpty()) -// .map(e -> e.stream().distinct().collect(Collectors.toList())) -// .map(om::writeValueAsString) -// .map(TextMessage::new) -// .doOnNext(msg -> System.out.println(msg.getPayload())) -// .subscribe(); -// -// List images = Arrays.asList(appStateService.createImage("http://imagezilla.net/show/1wovAmO-zt4_0002.jpg", "001", "test", imageZillaHost)); -// -// Post post = appStateService.createPost("test", "https://vipergirls.to/threads/4537276-Casey-Set-9057-5600px-94X-(unreleased)", images, new HashMap<>(), "001"); -// -// downloadQ.enqueue(Arrays.asList(post)); -// -//// Post post = appStateService.getPost("001"); -// -// post.getImages().get(0).init(appStateService); -// downloadQ.put(post.getImages().get(0)); -// -// post.getImages().get(0).init(appStateService); -// downloadQ.put(post.getImages().get(0)); -// -// post.getImages().get(0).init(appStateService); -// downloadQ.put(post.getImages().get(0)); + public void run(String... args) { + persistenceService.restore(); + appSettings.restore(); + + try { + authService.authenticate(); + } catch (VripperException e) { + logger.error("Cannot authenticate user with ViperGirls", e); + } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java b/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java index 94c2da8..638b8d2 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java @@ -42,49 +42,49 @@ public class Image { this.postId = postId; this.postName = postName; this.host = host; - this.status = Status.PENDING; this.appStateService = appStateService; - this.imageStateProcessor = BehaviorProcessor.create(); - this.appStateService.getCurrentImages().put(this.url, this); - this.appStateService.getAllImageState().onNext(this); - this.init(); + status = Status.PENDING; + imageStateProcessor = BehaviorProcessor.create(); + appStateService.getCurrentImages().put(this.url, this); + appStateService.getAllImageState().onNext(this); + init(); } public void setStatus(Status status) { this.status = status; - this.update(); + update(); } public boolean isCompleted() { - return this.status.equals(Status.COMPLETE); + return status.equals(Status.COMPLETE); } public void init() { - if (this.imageStateProcessor != null) { - this.imageStateProcessor.onComplete(); + if (imageStateProcessor != null) { + imageStateProcessor.onComplete(); } - if (this.subscription != null) { - this.subscription.dispose(); + if (subscription != null) { + subscription.dispose(); } - this.imageStateProcessor = BehaviorProcessor.create(); - this.subscription = imageStateProcessor + imageStateProcessor = BehaviorProcessor.create(); + subscription = imageStateProcessor .onBackpressureBuffer() .doOnNext(appStateService::onImageUpdate) .subscribe(); - this.current.set(0); - this.status = Status.PENDING; + current.set(0); + status = Status.PENDING; imageStateProcessor.onNext(this); } public void setCurrent(int current) { this.current.set(current); - this.update(); + update(); } public void increase(int read) { - this.current.addAndGet(read); - this.update(); + current.addAndGet(read); + update(); } private void update() { @@ -94,8 +94,8 @@ public class Image { imageStateProcessor.onNext(this); if (isCompleted()) { imageStateProcessor.onComplete(); - this.subscription.dispose(); - this.imageStateProcessor = null; + subscription.dispose(); + imageStateProcessor = null; } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java b/vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java index 875f7c9..bb4bca1 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java @@ -8,6 +8,7 @@ import tn.mnlr.vripper.services.AppStateService; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @Getter @@ -17,6 +18,7 @@ public class Post { @Setter private AppStateService appStateService; + private Status status; private final String type = "post"; @@ -33,30 +35,30 @@ public class Post { private AtomicInteger done = new AtomicInteger(0); + private int total; + public Post(String title, String url, List images, Map metadata, String postId, AppStateService appStateService) { this.title = title; this.url = url; this.images = images; this.metadata = metadata; this.postId = postId; - this.total = images.size(); - this.status = Status.PENDING; this.appStateService = appStateService; - this.appStateService.getCurrentPosts().put(postId, this); - this.appStateService.getSnapshotPostsState().onNext(this); + total = images.size(); + status = Status.PENDING; + appStateService.getCurrentPosts().put(postId, this); + appStateService.getSnapshotPostsState().onNext(this); } - private int total; - public void increase() { done.incrementAndGet(); - this.appStateService.getLivePostsState().onNext(this); + appStateService.getLivePostsState().onNext(this); } public void setStatus(Status status) { this.status = status; - if (this.appStateService != null) { - this.appStateService.getLivePostsState().onNext(this); + if (appStateService != null) { + appStateService.getLivePostsState().onNext(this); } } @@ -65,12 +67,15 @@ public class Post { } @Override - public int hashCode() { - return this.postId.hashCode(); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Post post = (Post) o; + return postId.equals(post.postId); } @Override - public boolean equals(Object obj) { - return this.postId.equals(obj); + public int hashCode() { + return Objects.hash(postId); } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java b/vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java index eaa6ede..e333229 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java @@ -4,4 +4,12 @@ public class HostException extends Exception { public HostException(Exception e) { super(e); } + + public HostException(String message) { + super(message); + } + + public HostException(String message, Exception e) { + super(message,e); + } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/exception/ValidationException.java b/vripper-server/src/main/java/tn/mnlr/vripper/exception/ValidationException.java index 60f1a30..17af339 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/exception/ValidationException.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/exception/ValidationException.java @@ -5,8 +5,4 @@ public class ValidationException extends Exception { public ValidationException(String message) { super(message); } - - public ValidationException(Exception e) { - super(e); - } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java index 5742282..975279f 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java @@ -8,25 +8,28 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Node; import tn.mnlr.vripper.exception.HostException; -import tn.mnlr.vripper.exception.HtmlProcessorException; import tn.mnlr.vripper.exception.XpathException; import tn.mnlr.vripper.q.ImageFileData; import tn.mnlr.vripper.services.ConnectionManager; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @Service public class AcidimgHost extends Host { - String host = "acidimg.cc"; - boolean https = true; + private static final Logger logger = LoggerFactory.getLogger(AcidimgHost.class); + + private static final String host = "acidimg.cc"; + private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']"; + public static final String IMG_XPATH = "//img[@class='centred']"; @Autowired private ConnectionManager cm; @@ -39,35 +42,20 @@ public class AcidimgHost extends Host { @Override protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException { - String basePage; + Document doc = getDocument(url); - HttpClient client = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(url); - - try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) { - basePage = EntityUtils.toString(response.getEntity()); - EntityUtils.consumeQuietly(response.getEntity()); - } catch (IOException e) { - throw new HostException(e); - } - - Document doc = null; + Node contDiv; try { - doc = htmlProcessorService.clean(basePage); - } catch (HtmlProcessorException e) { - throw new HostException(e); - } - - Node contDiv = null; - try { - contDiv = xpathService.getAsNode(doc, "//input[@id='continuebutton']"); + logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url)); + contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH); } catch (XpathException e) { - e.printStackTrace(); + throw new HostException(e); } if (contDiv != null) { - client = this.cm.getClient().build(); - HttpPost httpPost = this.cm.buildHttpPost(url); + logger.info(String.format("Click button found for %s", url)); + HttpClient client = cm.getClient().build(); + HttpPost httpPost = cm.buildHttpPost(url); httpPost.addHeader("Referer", url); List params = new ArrayList<>(); params.add(new BasicNameValuePair("imgContinue", "Continue to your image")); @@ -77,35 +65,37 @@ public class AcidimgHost extends Host { throw new HostException(e); } + logger.info(String.format("Requesting %s", httpPost)); try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) { + logger.info(String.format("Cleaning response for %s", httpPost)); doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity())); EntityUtils.consumeQuietly(response.getEntity()); - } catch (IOException | HtmlProcessorException e) { + } catch (Exception e) { throw new HostException(e); } } - Node imgNode = null; + Node imgNode; try { - imgNode = xpathService.getAsNode(doc, "//img[@class='centred']"); + logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url)); + imgNode = xpathService.getAsNode(doc, IMG_XPATH); } catch (XpathException e) { throw new HostException(e); } - String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim().concat(".jpg"); - String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); + if(imgNode == null) { + throw new HostException("Cannot find the image node"); + } - imageFileData.setImageUrl(imgUrl); - imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle); + try { + logger.info(String.format("Resolving name and image url for %s", url)); + String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(); + String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); - } - - @Override - protected void setImageRequest(ImageFileData imageFileData) throws IOException { - -// HttpClient client = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl()); - httpGet.addHeader("Referer", imageFileData.getPageUrl()); - imageFileData.setImageRequest(httpGet); + imageFileData.setImageUrl(imgUrl); + imageFileData.setImageName(imgTitle.isEmpty() ? getDefaultImageName(imgUrl) : imgTitle); + } catch (Exception e) { + throw new HostException("Unexpected error occurred", e); + } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java index 6325cc8..9af768b 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java @@ -2,13 +2,18 @@ package tn.mnlr.vripper.host; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.w3c.dom.Document; import tn.mnlr.vripper.AppSettings; import tn.mnlr.vripper.entities.Image; import tn.mnlr.vripper.exception.DownloadException; import tn.mnlr.vripper.exception.HostException; +import tn.mnlr.vripper.exception.HtmlProcessorException; import tn.mnlr.vripper.q.ImageFileData; import tn.mnlr.vripper.services.ConnectionManager; import tn.mnlr.vripper.services.HtmlProcessorService; @@ -23,6 +28,10 @@ import java.util.Random; @Service abstract public class Host { + private static final Logger logger = LoggerFactory.getLogger(Host.class); + + private static final int READ_BUFFER_SIZE = 8192; + @Autowired protected HtmlProcessorService htmlProcessorService; @@ -35,8 +44,6 @@ abstract public class Host { @Autowired private ConnectionManager cm; - int readBufferSize = 8192; - abstract protected String getHost(); public boolean isSupported(String url) { @@ -52,7 +59,12 @@ abstract public class Host { /** * HOST SPECIFIC */ + logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost())); setNameAndUrl(image.getUrl(), imageFileData); + logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName())); + logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl())); + + logger.info(String.format("Building image request for %s", image.getUrl())); setImageRequest(imageFileData); /** * END HOST SPECIFIC @@ -62,13 +74,18 @@ abstract public class Host { imageFileData.setImageName(imageFileData.getImageName() + ".jpg"); } - File destinationFolder = new File(appSettings.getDownloadPath(), this.sanitize(image.getPostName())); + File destinationFolder = new File(appSettings.getDownloadPath(), sanitize(image.getPostName())); + logger.info(String.format("Saving to %s", destinationFolder.getPath())); if (!destinationFolder.exists()) { + logger.info(String.format("Creating %s", destinationFolder.getPath())); destinationFolder.mkdirs(); + } else { + logger.warn(String.format("%s already exists, the file will be overridden", destinationFolder.getPath())); } - HttpClient client = this.cm.getClient().build(); + HttpClient client = cm.getClient().build(); + logger.info(String.format("Downloading %s", imageFileData.getImageUrl())); try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) { if(response.getStatusLine().getStatusCode() / 100 != 2) { @@ -82,11 +99,12 @@ abstract public class Host { ) { image.setTotal(response.getEntity().getContentLength()); + logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal())); + logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl())); - byte[] buffer = new byte[readBufferSize]; + byte[] buffer = new byte[READ_BUFFER_SIZE]; int read; - while ((read = downloadStream.read(buffer, 0, readBufferSize)) != -1) { -// randomFail(); + while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) { fos.write(buffer, 0, read); image.increase(read); } @@ -101,6 +119,10 @@ abstract public class Host { } } + /** + * Just for testing, you may ignore + * @throws Exception + */ private void randomFail() throws Exception { Random random = new Random(); int i = random.nextInt(100); @@ -109,11 +131,48 @@ abstract public class Host { } } - protected String sanitize(String folderName) { - return folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_"); + + + protected final String sanitize(final String folderName) { + String sanitizedFolderName = folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_"); + logger.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName)); + return sanitizedFolderName; } - protected abstract void setImageRequest(ImageFileData imageFileData) throws IOException; + protected final String getDefaultImageName(final String imgUrl) { + String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1); + logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle)); + return imgUrl; + } - protected abstract void setNameAndUrl(String url, ImageFileData imageFileData) throws HostException; + protected final Document getDocument(final String url) throws HostException { + String basePage; + + HttpClient client = cm.getClient().build(); + HttpGet httpGet = cm.buildHttpGet(url); + + logger.info(String.format("Requesting %s", url)); + try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) { + basePage = EntityUtils.toString(response.getEntity()); + logger.debug(String.format("%s response: %n%s", url, basePage)); + EntityUtils.consumeQuietly(response.getEntity()); + } catch (IOException e) { + throw new HostException(e); + } + + try { + logger.info(String.format("Cleaning %s response", url)); + return htmlProcessorService.clean(basePage); + } catch (HtmlProcessorException e) { + throw new HostException(e); + } + } + + protected void setImageRequest(final ImageFileData imageFileData) { + HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl()); + httpGet.addHeader("Referer", imageFileData.getPageUrl()); + imageFileData.setImageRequest(httpGet); + } + + protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException; } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java index fc5f93b..773833c 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java @@ -1,14 +1,13 @@ package tn.mnlr.vripper.host; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; +import org.w3c.dom.Node; import tn.mnlr.vripper.exception.HostException; -import tn.mnlr.vripper.exception.HtmlProcessorException; import tn.mnlr.vripper.exception.XpathException; import tn.mnlr.vripper.q.ImageFileData; import tn.mnlr.vripper.services.ConnectionManager; @@ -18,56 +17,47 @@ import java.io.IOException; @Service public class ImageZillaHost extends Host { + private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class); + + private static final String host = "imagezilla.net"; + public static final String IMG_XPATH = "//img[@id='photo']"; + @Autowired private ConnectionManager cm; - String host = "imagezilla.net"; - - boolean https = false; - @Override protected String getHost() { return host; } @Override - protected void setNameAndUrl(String url, ImageFileData imageFileData) throws HostException { + protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException { - String basePage; + Document doc = getDocument(url); - HttpClient client = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(url); - - try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) { - basePage = EntityUtils.toString(response.getEntity()); - EntityUtils.consumeQuietly(response.getEntity()); - } catch (IOException e) { - throw new HostException(e); - } - - Document doc = null; + String title; try { - doc = htmlProcessorService.clean(basePage); - } catch (HtmlProcessorException e) { - throw new HostException(e); - } - String title = null; - try { - title = xpathService.getAsNode(doc, "//img[@id='photo']").getAttributes().getNamedItem("title").getTextContent().trim(); + logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url)); + Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title"); + logger.info(String.format("Resolving name for %s", url)); + if(titleNode != null) { + title = titleNode.getTextContent().trim(); + } else { + title = null; + } } catch (XpathException e) { throw new HostException(e); } - imageFileData.setImageUrl(url.replace("show", "images")); - imageFileData.setImageName(title.substring(8)); + if(title == null || title.isEmpty()) { + title = getDefaultImageName(url); + } - } - - @Override - protected void setImageRequest(ImageFileData imageFileData) throws IOException { - - HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl()); - httpGet.addHeader("Referer", imageFileData.getPageUrl()); - imageFileData.setImageRequest(httpGet); + try { + imageFileData.setImageUrl(url.replace("show", "images")); + imageFileData.setImageName(title.substring(8)); + } catch (Exception e) { + throw new HostException("Unexpected error occurred", e); + } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java index c5e455d..3610b7e 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java @@ -1,26 +1,23 @@ package tn.mnlr.vripper.host; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Node; import tn.mnlr.vripper.exception.HostException; -import tn.mnlr.vripper.exception.HtmlProcessorException; import tn.mnlr.vripper.exception.XpathException; import tn.mnlr.vripper.q.ImageFileData; import tn.mnlr.vripper.services.ConnectionManager; -import java.io.IOException; - @Service public class ImgboxHost extends Host { - String host = "imgbox.com"; - boolean https = true; + private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class); + + private static final String host = "imgbox.com"; + public static final String IMG_XPATH = "//img[@id='img']"; @Autowired private ConnectionManager cm; @@ -33,45 +30,25 @@ public class ImgboxHost extends Host { @Override protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException { - String basePage; + Document doc = getDocument(url); - HttpClient client = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(url); - - try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) { - basePage = EntityUtils.toString(response.getEntity()); - EntityUtils.consumeQuietly(response.getEntity()); - } catch (IOException e) { - throw new HostException(e); - } - - Document doc = null; + Node imgNode; try { - doc = htmlProcessorService.clean(basePage); - } catch (HtmlProcessorException e) { - throw new HostException(e); - } - - Node imgNode = null; - try { - imgNode = xpathService.getAsNode(doc, "//img[@id='img']"); + logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url)); + imgNode = xpathService.getAsNode(doc, IMG_XPATH); } catch (XpathException e) { throw new HostException(e); } - String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim(); - String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); + try { + logger.info(String.format("Resolving name and image url for %s", url)); + String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim(); + String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); - imageFileData.setImageUrl(imgUrl); - imageFileData.setImageName(imgTitle.substring(8)); - - } - - @Override - protected void setImageRequest(ImageFileData imageFileData) throws IOException { - - HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl()); - httpGet.addHeader("Referer", imageFileData.getPageUrl()); - imageFileData.setImageRequest(httpGet); + imageFileData.setImageUrl(imgUrl); + imageFileData.setImageName(imgTitle.substring(8)); + } catch (Exception e) { + throw new HostException("Unexpected error occurred", e); + } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java index 28639f6..2b2f205 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java @@ -4,10 +4,11 @@ import org.apache.http.NameValuePair; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; @@ -25,8 +26,12 @@ import java.util.List; @Service public class ImxHost extends Host { - String host = "imx.to"; - boolean https = true; + private static final Logger logger = LoggerFactory.getLogger(ImxHost.class); + + private static final String host = "imx.to"; + public static final String CONTINUE_BUTTON_XPATH = "//div[@id='continuetoimage']"; + public static final String IMG_XPATH = "//img[@class='centred']"; + @Autowired private ConnectionManager cm; @@ -38,35 +43,20 @@ public class ImxHost extends Host { @Override protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException { - String basePage; + Document doc = getDocument(url); - HttpClient client = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(url); - - try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) { - basePage = EntityUtils.toString(response.getEntity()); - EntityUtils.consumeQuietly(response.getEntity()); - } catch (IOException e) { - throw new HostException(e); - } - - Document doc = null; + Node contDiv; try { - doc = htmlProcessorService.clean(basePage); - } catch (HtmlProcessorException e) { - throw new HostException(e); - } - - Node contDiv = null; - try { - contDiv = xpathService.getAsNode(doc, "//div[@id='continuetoimage']"); + logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url)); + contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH); } catch (XpathException e) { - e.printStackTrace(); + throw new HostException(e); } if (contDiv != null) { - client = this.cm.getClient().build(); - HttpPost httpPost = this.cm.buildHttpPost(url); + logger.info(String.format("Click button found for %s", url)); + HttpClient client = cm.getClient().build(); + HttpPost httpPost = cm.buildHttpPost(url); httpPost.addHeader("Referer", url); List params = new ArrayList<>(); params.add(new BasicNameValuePair("imgContinue", "Continue to image ... ")); @@ -75,8 +65,9 @@ public class ImxHost extends Host { } catch (Exception e) { throw new HostException(e); } - + logger.info(String.format("Requesting %s", httpPost)); try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) { + logger.info(String.format("Cleaning response for %s", httpPost)); doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity())); EntityUtils.consumeQuietly(response.getEntity()); } catch (IOException | HtmlProcessorException e) { @@ -84,26 +75,23 @@ public class ImxHost extends Host { } } - Node imgNode = null; + Node imgNode; try { - imgNode = xpathService.getAsNode(doc, "//img[@class='centred']"); + logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url)); + imgNode = xpathService.getAsNode(doc, IMG_XPATH); } catch (XpathException e) { throw new HostException(e); } - String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(); - String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); + try { + logger.info(String.format("Resolving name and image url for %s", url)); + String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(); + String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); - imageFileData.setImageUrl(imgUrl); - imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle); - - } - - @Override - protected void setImageRequest(ImageFileData imageFileData) throws IOException { - - HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl()); - httpGet.addHeader("Referer", imageFileData.getPageUrl()); - imageFileData.setImageRequest(httpGet); + imageFileData.setImageUrl(imgUrl); + imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle); + } catch(Exception e) { + throw new HostException("Unexpected error occurred", e); + } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java index 80733eb..ed751fa 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java @@ -1,26 +1,24 @@ package tn.mnlr.vripper.host; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Node; import tn.mnlr.vripper.exception.HostException; -import tn.mnlr.vripper.exception.HtmlProcessorException; import tn.mnlr.vripper.exception.XpathException; import tn.mnlr.vripper.q.ImageFileData; import tn.mnlr.vripper.services.ConnectionManager; -import java.io.IOException; - @Service public class PixhostHost extends Host { - String host = "pixhost.to"; - boolean https = false; + private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class); + + private static final String host = "pixhost.to"; + public static final String IMG_XPATH = "//img[@id='image']"; + @Autowired private ConnectionManager cm; @@ -32,45 +30,25 @@ public class PixhostHost extends Host { @Override protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException { - String basePage; + Document doc = getDocument(url); - HttpClient client = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(url); - - try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) { - basePage = EntityUtils.toString(response.getEntity()); - EntityUtils.consumeQuietly(response.getEntity()); - } catch (IOException e) { - throw new HostException(e); - } - - Document doc = null; + Node imgNode; try { - doc = htmlProcessorService.clean(basePage); - } catch (HtmlProcessorException e) { - throw new HostException(e); - } - - Node imgNode = null; - try { - imgNode = xpathService.getAsNode(doc, "//img[@id='image']"); + logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url)); + imgNode = xpathService.getAsNode(doc, IMG_XPATH); } catch (XpathException e) { throw new HostException(e); } - String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(); - String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); + try { + logger.info(String.format("Resolving name and image url for %s", url)); + String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(); + String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim(); - imageFileData.setImageUrl(imgUrl); - imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1)); - - } - - @Override - protected void setImageRequest(ImageFileData imageFileData) throws IOException { - - HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl()); - httpGet.addHeader("Referer", imageFileData.getPageUrl()); - imageFileData.setImageRequest(httpGet); + imageFileData.setImageUrl(imgUrl); + imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1)); + } catch (Exception e) { + throw new HostException("Unexpected error occurred", e); + } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java b/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java index a781326..edfcbc3 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java @@ -10,13 +10,13 @@ import java.util.concurrent.Callable; public class DownloadJob implements Callable { - private Logger logger = LoggerFactory.getLogger(DownloadJob.class); + private static final Logger logger = LoggerFactory.getLogger(DownloadJob.class); @Getter - final Image image; + private final Image image; @Getter - final ImageFileData imageFileData = new ImageFileData(); + private final ImageFileData imageFileData = new ImageFileData(); public DownloadJob(Image image) { this.image = image; @@ -25,12 +25,12 @@ public class DownloadJob implements Callable { @Override public Image call() throws Exception { - logger.debug(String.format("Starting downloading %s", image.getUrl())); - this.image.setStatus(Image.Status.DOWNLOADING); - this.image.setCurrent(0); - this.image.getHost().download(this.image, this.imageFileData); - this.image.setStatus(Image.Status.COMPLETE); - return this.image; + logger.info(String.format("Starting downloading %s", image.getUrl())); + image.setStatus(Image.Status.DOWNLOADING); + image.setCurrent(0); + image.getHost().download(image, imageFileData); + image.setStatus(Image.Status.COMPLETE); + return image; } @Override diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java b/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java index bcc42f2..9754f66 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java @@ -18,7 +18,7 @@ import java.util.stream.Collectors; @Service public class DownloadQ { - private Logger logger = LoggerFactory.getLogger(DownloadQ.class); + private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class); @Autowired private AppStateService appStateService; @@ -33,18 +33,18 @@ public class DownloadQ { public synchronized void put(Image image) { try { - logger.debug(String.format("Enqueuing a job for %s", image.getUrl())); + logger.info(String.format("Enqueuing a job for %s", image.getUrl())); DownloadJob downloadJob = new DownloadJob(image); - this.downloadQ.put(downloadJob); - this.appStateService.newDownloadJob(downloadJob); + downloadQ.put(downloadJob); + appStateService.newDownloadJob(downloadJob); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } public DownloadJob take() throws InterruptedException { - DownloadJob downloadJob = this.downloadQ.take(); - logger.debug(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl())); + DownloadJob downloadJob = downloadQ.take(); + logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl())); return downloadJob; } @@ -68,7 +68,7 @@ public class DownloadQ { return; } appStateService.getPost(postId).setStatus(Post.Status.PENDING); - logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId)); + logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId)); images.forEach(image -> { image.init(); put(image); @@ -77,16 +77,16 @@ public class DownloadQ { public void removeScheduled(Image image) { image.setStatus(Image.Status.STOPPED); - logger.debug(String.format("Removing scheduled job for %s", image.getUrl())); + logger.info(String.format("Removing scheduled job for %s", image.getUrl())); - Iterator iterator = this.downloadQ.iterator(); + Iterator iterator = downloadQ.iterator(); boolean removed = false; while(iterator.hasNext()) { DownloadJob next = iterator.next(); if(next.getImage().getPostId().equals(image.getPostId())) { iterator.remove(); appStateService.doneDownloadJob(image); - logger.debug(String.format("Scheduled job for %s is removed", image.getUrl())); + logger.info(String.format("Scheduled job for %s is removed", image.getUrl())); removed = true; break; } @@ -98,14 +98,14 @@ public class DownloadQ { } public void removeRunning(String postId) { - logger.debug(String.format("Interrupting running jobs for post id %s", postId)); - this.executionService.stop(postId); + logger.info(String.format("Interrupting running jobs for post id %s", postId)); + executionService.stop(postId); } public synchronized void stop(String postId) { try { - this.notPauseQ = false; + notPauseQ = false; appStateService.getPost(postId).setStatus(Post.Status.STOPPED); List images = appStateService.getPost(postId) .getImages() @@ -115,11 +115,11 @@ public class DownloadQ { if (images.isEmpty()) { return; } - logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId)); + logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId)); images.forEach(image -> removeScheduled(image)); removeRunning(postId); } finally { - this.notPauseQ = true; + notPauseQ = true; } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java b/vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java index 6d3b8cb..852604b 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; @Service public class ExecutionService { - Logger logger = LoggerFactory.getLogger(ExecutionService.class); + private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class); @Autowired private DownloadQ downloadQ; @@ -70,12 +70,12 @@ public class ExecutionService { } public void stop(String postId) { - List data = this.running + List data = running .stream() .filter(e -> e.getImage().getPostId().equals(postId)) .peek(e -> e.getImage().setStatus(Image.Status.STOPPED)) .collect(Collectors.toList()); - logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId)); + logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId)); data.forEach(e -> { futures.get(e.getImage().getUrl()).cancel(true); @@ -85,7 +85,7 @@ public class ExecutionService { boolean canRun() { boolean canRun = threadCount.get() < settings.getMaxThreads(); - if (canRun && this.downloadQ.isNotPauseQ()) { + if (canRun && downloadQ.isNotPauseQ()) { threadCount.incrementAndGet(); return true; } @@ -97,7 +97,7 @@ public class ExecutionService { if (canRun()) { DownloadJob take = null; try { - take = this.downloadQ.take(); + take = downloadQ.take(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -109,15 +109,15 @@ public class ExecutionService { Failsafe.with(retryPolicy) .onFailure(e -> { if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) { - logger.debug("Job successfully interrupted"); + logger.info("Job successfully interrupted"); return; } logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure()); finalTake.getImage().setStatus(Image.Status.ERROR); }) .onComplete(e -> { - this.appStateService.doneDownloadJob(finalTake.getImage()); - logger.debug(String.format("Finished downloading %s", finalTake.getImage().getUrl())); + appStateService.doneDownloadJob(finalTake.getImage()); + logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl())); synchronized (threadCount) { int i = threadCount.decrementAndGet(); running.remove(finalTake); @@ -127,7 +127,7 @@ public class ExecutionService { }) .get(finalTake::call); }; - logger.debug(String.format("Scheduling a job for %s", finalTake.getImage().getUrl())); + logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl())); futures.put(finalTake.getImage().getUrl(), executor.submit(task)); } else { synchronized (threadCount) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java b/vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java index 3d6b274..55795a4 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java @@ -2,7 +2,6 @@ package tn.mnlr.vripper.q; import lombok.Getter; import lombok.Setter; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; @Getter @@ -12,6 +11,5 @@ public class ImageFileData { private String pageUrl; private String imageName; private String imageUrl; -// private CloseableHttpResponse imageResponse; private HttpUriRequest imageRequest; } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java index 6101276..075a213 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java @@ -38,7 +38,7 @@ public class AppStateService { public void onImageUpdate(Image imageState) { - this.persistenceService.getProcessor().onNext(currentPosts); + persistenceService.getProcessor().onNext(currentPosts); liveImageUpdates.onNext(imageState); if (imageState.isCompleted()) { @@ -57,24 +57,24 @@ public class AppStateService { } public Post getPost(String postId) { - return this.currentPosts.get(postId); + return currentPosts.get(postId); } public synchronized void newDownloadJob(DownloadJob downloadJob) { String postId = downloadJob.getImage().getPostId(); checkKeyRunningPosts(postId); - int i = this.runningPosts.get(postId).incrementAndGet(); + int i = runningPosts.get(postId).incrementAndGet(); if (i > 0) { - Post post = this.currentPosts.get(postId); + Post post = currentPosts.get(postId); post.setStatus(Post.Status.DOWNLOADING); - this.livePostsState.onNext(post); + livePostsState.onNext(post); } } public void doneDownloadJob(Image image) { String postId = image.getPostId(); - int i = this.runningPosts.get(postId).decrementAndGet(); - Post post = this.currentPosts.get(postId); + int i = runningPosts.get(postId).decrementAndGet(); + Post post = currentPosts.get(postId); if(image.getStatus().equals(Image.Status.ERROR)) { post.setStatus(Post.Status.PARTIAL); } @@ -84,13 +84,13 @@ public class AppStateService { } else { post.setStatus(Post.Status.COMPLETE); } - this.livePostsState.onNext(post); + livePostsState.onNext(post); } } private synchronized void checkKeyRunningPosts(String key) { - if (!this.runningPosts.containsKey(key)) { - this.runningPosts.put(key, new AtomicInteger(0)); + if (!runningPosts.containsKey(key)) { + runningPosts.put(key, new AtomicInteger(0)); } } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java index 2fced89..2617fef 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java @@ -10,12 +10,11 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.springframework.stereotype.Service; - @Service public class ConnectionManager { private ConnectionManager() { - this.buildConnectionPool(); + buildConnectionPool(); } private PoolingHttpClientConnectionManager pcm; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java index dc5d6cc..e40aea8 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java @@ -7,7 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import tn.mnlr.vripper.Main; +import tn.mnlr.vripper.VripperApplication; import tn.mnlr.vripper.entities.Image; import tn.mnlr.vripper.entities.Post; import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin; @@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit; @Service public class PersistenceService { - private Logger logger = LoggerFactory.getLogger(PersistenceService.class); + private static final Logger logger = LoggerFactory.getLogger(PersistenceService.class); @Autowired private AppStateService stateService; @@ -48,7 +48,7 @@ public class PersistenceService { } public void persist(Map currentPosts) { - try(PrintWriter out = new PrintWriter(Main.dataPath)) { + try(PrintWriter out = new PrintWriter(VripperApplication.dataPath)) { out.print(om.writeValueAsString(currentPosts)); } catch (IOException e) { logger.error("Failed to persist app state", e); @@ -70,13 +70,13 @@ public class PersistenceService { String jsonContent; try { - jsonContent = new String(Files.readAllBytes(Paths.get(Main.dataPath))); + jsonContent = new String(Files.readAllBytes(Paths.get(VripperApplication.dataPath))); } catch (Exception e) { logger.warn("data file not found, previous state cannot be restored", e); return; } - Map read = this.read(jsonContent); + Map read = read(jsonContent); stateService.getCurrentPosts().clear(); stateService.getCurrentPosts().putAll(read); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java index b23805b..418af55 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java @@ -23,13 +23,14 @@ import java.util.List; @Service public class PostParser { + private static final Logger logger = LoggerFactory.getLogger(PostParser.class); + private final static String VIPER_GIRLS_BASE_ADRESS = "https://vipergirls.to/"; private final static String POSTS_XPATH = "//li[contains(@id,'post_')][not(contains(@id,'post_thank'))]"; private final static String REAL_THREAD_XPATH = ".//a[@class='postcounter']"; private final static String THREAD_TITLE_XPATH = "//li[contains(@class, 'lastnavbit')]/span"; private final static String POST_TITLE_XPATH = ".//h2"; private final static String POST_LINKS_XPATH = ".//a"; - private Logger logger = LoggerFactory.getLogger(PostParser.class); @Autowired private ConnectionManager cm; @@ -48,14 +49,14 @@ public class PostParser { public List parse(String threadUrl) throws PostParseException { - logger.debug(String.format("Parsing thread %s", threadUrl)); + logger.info(String.format("Parsing thread %s", threadUrl)); List posts = new ArrayList<>(); String postResponse; - HttpClient connection = this.cm.getClient().build(); - HttpGet httpGet = this.cm.buildHttpGet(threadUrl); + HttpClient connection = cm.getClient().build(); + HttpGet httpGet = cm.buildHttpGet(threadUrl); - logger.debug(String.format("Requesting %s", httpGet)); + logger.info(String.format("Requesting %s", httpGet)); try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) { if (response.getStatusLine().getStatusCode() / 100 != 2) { throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet)); @@ -69,7 +70,7 @@ public class PostParser { Document document; try { - logger.debug(String.format("Cleaning HTML response for XML parsing: %s", postResponse)); + logger.info(String.format("Cleaning HTML response for XML parsing: %s", postResponse)); document = htmlProcessorService.clean(postResponse); } catch (Exception e) { throw new PostParseException(e); @@ -77,11 +78,11 @@ public class PostParser { String threadTitle = null; try { - logger.debug(String.format("Looking for thread title using xpath: %s", THREAD_TITLE_XPATH)); + logger.info(String.format("Looking for thread title using xpath: %s", THREAD_TITLE_XPATH)); Node threadTitleNode = xpathService.getAsNode(document, THREAD_TITLE_XPATH); if (threadTitleNode != null) { threadTitle = threadTitleNode.getTextContent().trim(); - logger.debug(String.format("Thread title found %s", threadTitle)); + logger.info(String.format("Thread title found %s", threadTitle)); } if (threadTitle == null) { logger.warn("Cannot find thread's title"); @@ -92,33 +93,33 @@ public class PostParser { NodeList postsNodeList; try { - logger.debug(String.format("Looking for posts using xpath: %s", POSTS_XPATH)); + logger.info(String.format("Looking for posts using xpath: %s", POSTS_XPATH)); postsNodeList = xpathService.getAsNodeList(document, POSTS_XPATH); } catch (Exception e) { throw new PostParseException(e); } - logger.debug(String.format("Found %d posts in thread %s", postsNodeList.getLength(), threadUrl)); + logger.info(String.format("Found %d posts in thread %s", postsNodeList.getLength(), threadUrl)); for (int i = 0; i < postsNodeList.getLength(); i++) { String realUrl; - logger.debug(String.format("Parsing post #%d", i + 1)); + logger.info(String.format("Parsing post #%d", i + 1)); try { - logger.debug(String.format("Finding posts's link")); + logger.info(String.format("Finding posts's link")); realUrl = VIPER_GIRLS_BASE_ADRESS.concat(xpathService .getAsNode(postsNodeList.item(i), REAL_THREAD_XPATH) .getAttributes() .getNamedItem("href") .getTextContent() .trim()); - logger.debug(String.format("Post's link: %s", realUrl)); + logger.info(String.format("Post's link: %s", realUrl)); } catch (Exception e) { throw new PostParseException(e); } - logger.debug("Finding posts's id"); + logger.info("Finding posts's id"); String postId = realUrl.substring(realUrl.indexOf("#")).replace("#post", ""); - logger.debug(String.format("Post's id: %s", postId)); + logger.info(String.format("Post's id: %s", postId)); if (appStateService.getCurrentPosts().containsKey(postId)) { logger.warn(String.format("Post with id %s is already loaded, skipping", postId)); @@ -127,18 +128,18 @@ public class PostParser { String postTitle; try { - logger.debug(String.format("Finding post's title")); + logger.info(String.format("Finding post's title")); Node titleNode = xpathService.getAsNode(postsNodeList.item(i), POST_TITLE_XPATH); if (titleNode != null) { postTitle = titleNode.getTextContent().trim(); - logger.debug(String.format("Found post's title: %s", postTitle)); + logger.info(String.format("Found post's title: %s", postTitle)); } else { - logger.debug("Cannot find post's title"); + logger.info("Cannot find post's title"); if (threadTitle != null) { - logger.debug("Falling back to thread title to generate a post name"); + logger.info("Falling back to thread title to generate a post name"); postTitle = threadTitle + "#" + postId; } else { - logger.debug("Falling back to post id to generate a post name"); + logger.info("Falling back to post id to generate a post name"); postTitle = "#" + postId; } } @@ -148,7 +149,7 @@ public class PostParser { ArrayList imagesList = new ArrayList<>(); try { - logger.debug(String.format("Finding all links for post with id %s using xpath %s", postId, POST_LINKS_XPATH)); + logger.info(String.format("Finding all links for post with id %s using xpath %s", postId, POST_LINKS_XPATH)); NodeList imagesNodeList = xpathService.getAsNodeList(postsNodeList.item(i), POST_LINKS_XPATH); for (int j = 0; j < imagesNodeList.getLength(); j++) { @@ -156,14 +157,14 @@ public class PostParser { Host foundHost; if (imageHref != null) { String imageUrl = imageHref.getTextContent().trim(); - logger.debug(String.format("Scanning %s", imageUrl)); + logger.info(String.format("Scanning %s", imageUrl)); foundHost = supportedHosts.stream().filter(host -> host.isSupported(imageUrl)).findFirst().orElse(null); } else { logger.warn("href is null, skipping"); continue; } if (foundHost != null) { - logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref)); + logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref)); imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost)); } else { logger.warn(String.format("unsupported host for %s, skipping", imageHref)); @@ -175,7 +176,7 @@ public class PostParser { } if (!imagesList.isEmpty()) { - logger.debug(String.format("Found %d images for post with id %s", imagesList.size(), postId)); + logger.info(String.format("Found %d images for post with id %s", imagesList.size(), postId)); posts.add(appStateService.createPost(postTitle, realUrl, imagesList, null, postId)); } else { logger.warn(String.format("No images found for post with id %s, skipping", postId)); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java index 840e8f4..b109cd3 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java @@ -23,7 +23,7 @@ import java.util.List; @Service public class VipergirlsAuthService { - private Logger logger = LoggerFactory.getLogger(VipergirlsAuthService.class); + private static final Logger logger = LoggerFactory.getLogger(VipergirlsAuthService.class); @Autowired private ConnectionManager cm; @@ -44,11 +44,11 @@ public class VipergirlsAuthService { public void authenticate() throws VripperException { logger.info("Authenticating using ViperGirls credentials"); - this.authenticated = false; + authenticated = false; if(!appSettings.isVLogin()) { logger.warn("Authentication option is disabled"); - this.cookies = null; + cookies = null; return; } @@ -102,7 +102,7 @@ public class VipergirlsAuthService { } catch (Exception e) { throw new VripperException(e); } - this.authenticated = true; + authenticated = true; logger.info(String.format("Authenticated: %s", username)); } @@ -143,7 +143,7 @@ public class VipergirlsAuthService { String postPage = EntityUtils.toString(response.getEntity()); Document document = htmlProcessorService.clean(postPage); - String thanksUrl = this.xpathService + String thanksUrl = xpathService .getAsNode(document, "//li[contains(@id,'post_')][not(contains(@id,'post_thank'))]//a[@class='post_thanks_button']") .getAttributes() .getNamedItem("href") diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java index 76619e3..6876195 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java @@ -21,7 +21,8 @@ import java.util.List; @CrossOrigin(value = "*") public class PostRestEndpoint { - Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class); + private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class); + @Autowired private AppStateService appStateService; @@ -48,14 +49,14 @@ public class PostRestEndpoint { @PostMapping("/post") @ResponseStatus(value = HttpStatus.OK) public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception { - this.logger.info(String.format("Starting to process thread: %s", url.url)); + logger.info(String.format("Starting to process thread: %s", url.url)); if (url.url == null || url.url.isEmpty()) { - return new ResponseEntity("Failed to process empty request", HttpStatus.BAD_REQUEST); + return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST); } else if (!url.url.startsWith("https://vipergirls.to")) { - return new ResponseEntity("ViperGirls only links are supported", HttpStatus.BAD_REQUEST); + return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST); } - List parsed = this.postParser.parse(url.url); - this.logger.debug(String.format("%d posts found from thread %s", parsed.size(), url.url)); + List parsed = postParser.parse(url.url); + logger.info(String.format("%d posts found from thread %s", parsed.size(), url.url)); parsed.forEach(p -> { try { authService.leaveThanks(p.getUrl(), p.getPostId()); @@ -63,28 +64,28 @@ public class PostRestEndpoint { logger.error(String.format("Failed to leave thanks for %s", p.getUrl()), e); } }); - if(this.appSettings.isAutoStart()) { - this.logger.info("Auto start downloads option is enabled"); - this.logger.debug(String.format("Starting to enqueue %d jobs for %s", parsed.stream().flatMap(e -> e.getImages().stream()).count(), url.url)); - parsed.forEach(post -> this.downloadQ.enqueue(post)); - this.logger.debug(String.format("Done enqueuing jobs for %s", url.url)); + if(appSettings.isAutoStart()) { + logger.info("Auto start downloads option is enabled"); + logger.info(String.format("Starting to enqueue %d jobs for %s", parsed.stream().flatMap(e -> e.getImages().stream()).count(), url.url)); + parsed.forEach(post -> downloadQ.enqueue(post)); + logger.info(String.format("Done enqueuing jobs for %s", url.url)); } else { - this.logger.info("Auto start downloads option is disabled"); + logger.info("Auto start downloads option is disabled"); } - this.logger.info(String.format("Done processing thread: %s", url.url)); + logger.info(String.format("Done processing thread: %s", url.url)); return new ResponseEntity(HttpStatus.OK); } @PostMapping("/post/restart") @ResponseStatus(value = HttpStatus.OK) public void restartPost(@RequestBody PostId postId) throws Exception { - this.downloadQ.restart(postId.getPostId()); + downloadQ.restart(postId.getPostId()); } @PostMapping("/post/stop") @ResponseStatus(value = HttpStatus.OK) public void stop(@RequestBody PostId postId) throws Exception { - this.downloadQ.stop(postId.getPostId()); + downloadQ.stop(postId.getPostId()); } @Getter diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java index b52b447..eaf71ea 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java @@ -15,7 +15,7 @@ import tn.mnlr.vripper.services.VipergirlsAuthService; @CrossOrigin(value = "*") public class SettingsRestEndpoint { - private Logger logger = LoggerFactory.getLogger(SettingsRestEndpoint.class); + private static final Logger logger = LoggerFactory.getLogger(SettingsRestEndpoint.class); @Autowired private AppSettings settings; @@ -68,13 +68,13 @@ public class SettingsRestEndpoint { public AppSettings.Settings getSettings() throws Exception { return new AppSettings.Settings( - this.settings.getDownloadPath(), - this.settings.getMaxThreads(), - this.settings.isAutoStart(), - this.settings.isVLogin(), - this.settings.getVUsername(), - this.settings.getVPassword(), - this.settings.isVThanks() + settings.getDownloadPath(), + settings.getMaxThreads(), + settings.isAutoStart(), + settings.isVLogin(), + settings.getVUsername(), + settings.getVPassword(), + settings.isVThanks() ); } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java index 0a0938f..010b853 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; @Component public class WebSocketHandler extends TextWebSocketHandler { - private Logger logger = LoggerFactory.getLogger(WebSocketHandler.class); + private static final Logger logger = LoggerFactory.getLogger(WebSocketHandler.class); @Getter private static class WSMessage { @@ -69,11 +69,11 @@ public class WebSocketHandler extends TextWebSocketHandler { subscribeForPostDetails(session, wsMessage.getPayload()); break; case POST_DETAILS_UNSUB: - logger.debug("unsubscribe from post details"); + logger.info(String.format("Client %s unsubscribed from post details", session.getId())); Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose()); break; case POSTS_UNSUB: - logger.debug("unsubscribe from posts"); + logger.info(String.format("Client %s unsubscribed from posts", session.getId())); Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose()); break; } @@ -81,7 +81,7 @@ public class WebSocketHandler extends TextWebSocketHandler { private void subscribeForPosts(WebSocketSession session) { - logger.debug(String.format("Client %s subscribed for posts", session.getId())); + logger.info(String.format("Client %s subscribed for posts", session.getId())); if (postsSubscriptions.containsKey(session.getId())) { postsSubscriptions.get(session.getId()).dispose(); } @@ -93,13 +93,13 @@ public class WebSocketHandler extends TextWebSocketHandler { .map(e -> e.stream().distinct().collect(Collectors.toList())) .map(om::writeValueAsString) .map(TextMessage::new) - .subscribe(msg -> this.send(session, msg), e -> logger.error("Failed to send data to client", e)) + .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e)) ); } private void subscribeForPostDetails(WebSocketSession session, String postId) { - logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId)); + logger.info(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId)); if (postDetailsSubscriptions.containsKey(session.getId())) { postDetailsSubscriptions.get(session.getId()).dispose(); } @@ -112,7 +112,7 @@ public class WebSocketHandler extends TextWebSocketHandler { .map(e -> e.stream().distinct().collect(Collectors.toList())) .map(om::writeValueAsString) .map(TextMessage::new) - .subscribe(msg -> this.send(session, msg), e -> logger.error("Failed to send data to client", e)) + .subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e)) ); } diff --git a/vripper-server/src/main/resources/application.properties b/vripper-server/src/main/resources/application.properties index 45c714c..d6b3c36 100644 --- a/vripper-server/src/main/resources/application.properties +++ b/vripper-server/src/main/resources/application.properties @@ -1,3 +1,3 @@ -logging.level.org.springframework.web=DEBUG -logging.level.root=DEBUG -logging.level.org.apache.http=WARN \ No newline at end of file +logging.level.org.springframework.web=ERROR +logging.level.root=ERROR +logging.level.org.apache.http=ERROR \ No newline at end of file