External/vripper-project
1
0
Fork 0
mirror of https://github.com/dev-claw/vripper-project.git synced 2026-08-19 08:35:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup code and enhancements

Browse Source
This commit is contained in:
death-claw
2019-06-22 14:59:09 +01:00
parent e31a930402
commit 6ead2631fb
26 changed files with 397 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vripper-server/src/main/java/tn/mnlr/vripper/AppSettings.java
+15 -15
View File
@@ -21,7 +21,7 @@ import java.util.prefs.Preferences;
@Setter
public class AppSettings {
private Logger logger = LoggerFactory.getLogger(AppSettings.class);
private static final Logger logger = LoggerFactory.getLogger(AppSettings.class);
private Preferences prefs = Preferences.userNodeForPackage(tn.mnlr.vripper.AppSettings.class);
@@ -51,24 +51,24 @@ public class AppSettings {
public void restore() {
this.downloadPath = prefs.get(DOWNLOAD_PATH, "");
this.maxThreads = prefs.getInt(MAX_THREADS, 1);
this.autoStart = prefs.getBoolean(AUTO_START, false);
this.vLogin = prefs.getBoolean(V_LOGIN, false);
this.vUsername = prefs.get(V_USERNAME, "");
this.vPassword = prefs.get(V_PASSWORD, "");
this.vThanks = prefs.getBoolean(V_THANKS, false);
downloadPath = prefs.get(DOWNLOAD_PATH, "");
maxThreads = prefs.getInt(MAX_THREADS, 1);
autoStart = prefs.getBoolean(AUTO_START, false);
vLogin = prefs.getBoolean(V_LOGIN, false);
vUsername = prefs.get(V_USERNAME, "");
vPassword = prefs.get(V_PASSWORD, "");
vThanks = prefs.getBoolean(V_THANKS, false);
}
public void save() {
prefs.put(DOWNLOAD_PATH, this.downloadPath);
prefs.putInt(MAX_THREADS, this.maxThreads);
prefs.putBoolean(AUTO_START, this.autoStart);
prefs.putBoolean(V_LOGIN, this.vLogin);
prefs.put(V_USERNAME, this.vUsername);
prefs.put(V_PASSWORD, this.vPassword);
prefs.putBoolean(V_THANKS, this.vThanks);
prefs.put(DOWNLOAD_PATH, downloadPath);
prefs.putInt(MAX_THREADS, maxThreads);
prefs.putBoolean(AUTO_START, autoStart);
prefs.putBoolean(V_LOGIN, vLogin);
prefs.put(V_USERNAME, vUsername);
prefs.put(V_PASSWORD, vPassword);
prefs.putBoolean(V_THANKS, vThanks);
try {
prefs.sync();
vripper-server/src/main/java/tn/mnlr/vripper/Main.java
-47
View File
@@ -1,47 +0,0 @@
package tn.mnlr.vripper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.services.AppStateService;
import tn.mnlr.vripper.services.PersistenceService;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import java.io.File;
@Component
public class Main implements ApplicationRunner {
private Logger logger = LoggerFactory.getLogger(Main.class);
public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json";
@Autowired
private VipergirlsAuthService authService;
@Autowired
private PersistenceService persistenceService;
@Autowired
private AppStateService stateService;
@Autowired
private AppSettings appSettings;
@Override
public void run(ApplicationArguments args) {
persistenceService.restore();
appSettings.restore();
try {
authService.authenticate();
} catch (VripperException e) {
logger.error("Cannot authenticate user with ViperGirls", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/VripperApplication.java
+23 -39
View File
@@ -1,18 +1,23 @@
package tn.mnlr.vripper;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.host.ImageZillaHost;
import tn.mnlr.vripper.q.DownloadQ;
import tn.mnlr.vripper.services.AppStateService;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.services.PersistenceService;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import java.io.File;
@SpringBootApplication
public class VripperApplication {
public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json";
public static void main(String[] args) {
SpringApplication.run(VripperApplication.class, args);
}
@@ -20,48 +25,27 @@ public class VripperApplication {
@Component
public class AppCommandRunner implements CommandLineRunner {
@Autowired
private AppStateService appStateService;
private final Logger logger = LoggerFactory.getLogger(AppCommandRunner.class);
@Autowired
private ImageZillaHost imageZillaHost;
private VipergirlsAuthService authService;
@Autowired
private DownloadQ downloadQ;
private PersistenceService persistenceService;
private ObjectMapper om = new ObjectMapper();
@Autowired
private AppSettings appSettings;
@Override
public void run(String... args) throws Exception {
// om.addMixIn(Image.class, ImageUIMixin.class).addMixIn(Post.class, PostUIMixin.class);
//
// Disposable subscription = FlowableProcessor.merge(appStateService.getAllImageState(), appStateService.getLiveImageUpdates())
// .observeOn(Schedulers.io())
// .filter(e -> e.getPostId().equals("001"))
// .buffer(2, TimeUnit.SECONDS)
// .filter(e -> !e.isEmpty())
// .map(e -> e.stream().distinct().collect(Collectors.toList()))
// .map(om::writeValueAsString)
// .map(TextMessage::new)
// .doOnNext(msg -> System.out.println(msg.getPayload()))
// .subscribe();
//
// List<Image> images = Arrays.asList(appStateService.createImage("http://imagezilla.net/show/1wovAmO-zt4_0002.jpg", "001", "test", imageZillaHost));
//
// Post post = appStateService.createPost("test", "https://vipergirls.to/threads/4537276-Casey-Set-9057-5600px-94X-(unreleased)", images, new HashMap<>(), "001");
//
// downloadQ.enqueue(Arrays.asList(post));
//
//// Post post = appStateService.getPost("001");
//
// post.getImages().get(0).init(appStateService);
// downloadQ.put(post.getImages().get(0));
//
// post.getImages().get(0).init(appStateService);
// downloadQ.put(post.getImages().get(0));
//
// post.getImages().get(0).init(appStateService);
// downloadQ.put(post.getImages().get(0));
public void run(String... args) {
persistenceService.restore();
appSettings.restore();
try {
authService.authenticate();
} catch (VripperException e) {
logger.error("Cannot authenticate user with ViperGirls", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/entities/Image.java
+20 -20
View File
@@ -42,49 +42,49 @@ public class Image {
this.postId = postId;
this.postName = postName;
this.host = host;
this.status = Status.PENDING;
this.appStateService = appStateService;
this.imageStateProcessor = BehaviorProcessor.create();
this.appStateService.getCurrentImages().put(this.url, this);
this.appStateService.getAllImageState().onNext(this);
this.init();
status = Status.PENDING;
imageStateProcessor = BehaviorProcessor.create();
appStateService.getCurrentImages().put(this.url, this);
appStateService.getAllImageState().onNext(this);
init();
}
public void setStatus(Status status) {
this.status = status;
this.update();
update();
}
public boolean isCompleted() {
return this.status.equals(Status.COMPLETE);
return status.equals(Status.COMPLETE);
}
public void init() {
if (this.imageStateProcessor != null) {
this.imageStateProcessor.onComplete();
if (imageStateProcessor != null) {
imageStateProcessor.onComplete();
}
if (this.subscription != null) {
this.subscription.dispose();
if (subscription != null) {
subscription.dispose();
}
this.imageStateProcessor = BehaviorProcessor.create();
this.subscription = imageStateProcessor
imageStateProcessor = BehaviorProcessor.create();
subscription = imageStateProcessor
.onBackpressureBuffer()
.doOnNext(appStateService::onImageUpdate)
.subscribe();
this.current.set(0);
this.status = Status.PENDING;
current.set(0);
status = Status.PENDING;
imageStateProcessor.onNext(this);
}
public void setCurrent(int current) {
this.current.set(current);
this.update();
update();
}
public void increase(int read) {
this.current.addAndGet(read);
this.update();
current.addAndGet(read);
update();
}
private void update() {
@@ -94,8 +94,8 @@ public class Image {
imageStateProcessor.onNext(this);
if (isCompleted()) {
imageStateProcessor.onComplete();
this.subscription.dispose();
this.imageStateProcessor = null;
subscription.dispose();
imageStateProcessor = null;
}
}
vripper-server/src/main/java/tn/mnlr/vripper/entities/Post.java
+18 -13
View File
@@ -8,6 +8,7 @@ import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
@Getter
@@ -17,6 +18,7 @@ public class Post {
@Setter
private AppStateService appStateService;
private Status status;
private final String type = "post";
@@ -33,30 +35,30 @@ public class Post {
private AtomicInteger done = new AtomicInteger(0);
private int total;
public Post(String title, String url, List<Image> images, Map<String, String> metadata, String postId, AppStateService appStateService) {
this.title = title;
this.url = url;
this.images = images;
this.metadata = metadata;
this.postId = postId;
this.total = images.size();
this.status = Status.PENDING;
this.appStateService = appStateService;
this.appStateService.getCurrentPosts().put(postId, this);
this.appStateService.getSnapshotPostsState().onNext(this);
total = images.size();
status = Status.PENDING;
appStateService.getCurrentPosts().put(postId, this);
appStateService.getSnapshotPostsState().onNext(this);
}
private int total;
public void increase() {
done.incrementAndGet();
this.appStateService.getLivePostsState().onNext(this);
appStateService.getLivePostsState().onNext(this);
}
public void setStatus(Status status) {
this.status = status;
if (this.appStateService != null) {
this.appStateService.getLivePostsState().onNext(this);
if (appStateService != null) {
appStateService.getLivePostsState().onNext(this);
}
}
@@ -65,12 +67,15 @@ public class Post {
}
@Override
public int hashCode() {
return this.postId.hashCode();
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return postId.equals(post.postId);
}
@Override
public boolean equals(Object obj) {
return this.postId.equals(obj);
public int hashCode() {
return Objects.hash(postId);
}
}
vripper-server/src/main/java/tn/mnlr/vripper/exception/HostException.java
+8
View File
@@ -4,4 +4,12 @@ public class HostException extends Exception {
public HostException(Exception e) {
super(e);
}
public HostException(String message) {
super(message);
}
public HostException(String message, Exception e) {
super(message,e);
}
}
vripper-server/src/main/java/tn/mnlr/vripper/exception/ValidationException.java
-4
View File
@@ -5,8 +5,4 @@ public class ValidationException extends Exception {
public ValidationException(String message) {
super(message);
}
public ValidationException(Exception e) {
super(e);
}
}
vripper-server/src/main/java/tn/mnlr/vripper/host/AcidimgHost.java
+33 -43
View File
@@ -8,25 +8,28 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@Service
public class AcidimgHost extends Host {
String host = "acidimg.cc";
boolean https = true;
private static final Logger logger = LoggerFactory.getLogger(AcidimgHost.class);
private static final String host = "acidimg.cc";
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
public static final String IMG_XPATH = "//img[@class='centred']";
@Autowired
private ConnectionManager cm;
@@ -39,35 +42,20 @@ public class AcidimgHost extends Host {
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
String basePage;
Document doc = getDocument(url);
HttpClient client = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(url);
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
basePage = EntityUtils.toString(response.getEntity());
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
Document doc = null;
Node contDiv;
try {
doc = htmlProcessorService.clean(basePage);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
Node contDiv = null;
try {
contDiv = xpathService.getAsNode(doc, "//input[@id='continuebutton']");
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
e.printStackTrace();
throw new HostException(e);
}
if (contDiv != null) {
client = this.cm.getClient().build();
HttpPost httpPost = this.cm.buildHttpPost(url);
logger.info(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
httpPost.addHeader("Referer", url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
@@ -77,35 +65,37 @@ public class AcidimgHost extends Host {
throw new HostException(e);
}
logger.info(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
logger.info(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
} catch (Exception e) {
throw new HostException(e);
}
}
Node imgNode = null;
Node imgNode;
try {
imgNode = xpathService.getAsNode(doc, "//img[@class='centred']");
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim().concat(".jpg");
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
if(imgNode == null) {
throw new HostException("Cannot find the image node");
}
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
try {
logger.info(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
}
@Override
protected void setImageRequest(ImageFileData imageFileData) throws IOException {
// HttpClient client = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? getDefaultImageName(imgUrl) : imgTitle);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/host/Host.java
+70 -11
View File
@@ -2,13 +2,18 @@ package tn.mnlr.vripper.host;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.AppSettings;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HtmlProcessorService;
@@ -23,6 +28,10 @@ import java.util.Random;
@Service
abstract public class Host {
private static final Logger logger = LoggerFactory.getLogger(Host.class);
private static final int READ_BUFFER_SIZE = 8192;
@Autowired
protected HtmlProcessorService htmlProcessorService;
@@ -35,8 +44,6 @@ abstract public class Host {
@Autowired
private ConnectionManager cm;
int readBufferSize = 8192;
abstract protected String getHost();
public boolean isSupported(String url) {
@@ -52,7 +59,12 @@ abstract public class Host {
/**
* HOST SPECIFIC
*/
logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
setNameAndUrl(image.getUrl(), imageFileData);
logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
logger.info(String.format("Building image request for %s", image.getUrl()));
setImageRequest(imageFileData);
/**
* END HOST SPECIFIC
@@ -62,13 +74,18 @@ abstract public class Host {
imageFileData.setImageName(imageFileData.getImageName() + ".jpg");
}
File destinationFolder = new File(appSettings.getDownloadPath(), this.sanitize(image.getPostName()));
File destinationFolder = new File(appSettings.getDownloadPath(), sanitize(image.getPostName()));
logger.info(String.format("Saving to %s", destinationFolder.getPath()));
if (!destinationFolder.exists()) {
logger.info(String.format("Creating %s", destinationFolder.getPath()));
destinationFolder.mkdirs();
} else {
logger.warn(String.format("%s already exists, the file will be overridden", destinationFolder.getPath()));
}
HttpClient client = this.cm.getClient().build();
HttpClient client = cm.getClient().build();
logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
if(response.getStatusLine().getStatusCode() / 100 != 2) {
@@ -82,11 +99,12 @@ abstract public class Host {
) {
image.setTotal(response.getEntity().getContentLength());
logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
byte[] buffer = new byte[readBufferSize];
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, readBufferSize)) != -1) {
// randomFail();
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
fos.write(buffer, 0, read);
image.increase(read);
}
@@ -101,6 +119,10 @@ abstract public class Host {
}
}
/**
* Just for testing, you may ignore
* @throws Exception
*/
private void randomFail() throws Exception {
Random random = new Random();
int i = random.nextInt(100);
@@ -109,11 +131,48 @@ abstract public class Host {
}
}
protected String sanitize(String folderName) {
return folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
protected final String sanitize(final String folderName) {
String sanitizedFolderName = folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
logger.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
return sanitizedFolderName;
}
protected abstract void setImageRequest(ImageFileData imageFileData) throws IOException;
protected final String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
protected abstract void setNameAndUrl(String url, ImageFileData imageFileData) throws HostException;
protected final Document getDocument(final String url) throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
logger.info(String.format("Requesting %s", url));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
basePage = EntityUtils.toString(response.getEntity());
logger.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Cleaning %s response", url));
return htmlProcessorService.clean(basePage);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
protected void setImageRequest(final ImageFileData imageFileData) {
HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
}
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
}
vripper-server/src/main/java/tn/mnlr/vripper/host/ImageZillaHost.java
+28 -38
View File
@@ -1,14 +1,13 @@
package tn.mnlr.vripper.host;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
@@ -18,56 +17,47 @@ import java.io.IOException;
@Service
public class ImageZillaHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
private static final String host = "imagezilla.net";
public static final String IMG_XPATH = "//img[@id='photo']";
@Autowired
private ConnectionManager cm;
String host = "imagezilla.net";
boolean https = false;
@Override
protected String getHost() {
return host;
}
@Override
protected void setNameAndUrl(String url, ImageFileData imageFileData) throws HostException {
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
String basePage;
Document doc = getDocument(url);
HttpClient client = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(url);
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
basePage = EntityUtils.toString(response.getEntity());
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
Document doc = null;
String title;
try {
doc = htmlProcessorService.clean(basePage);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
String title = null;
try {
title = xpathService.getAsNode(doc, "//img[@id='photo']").getAttributes().getNamedItem("title").getTextContent().trim();
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
logger.info(String.format("Resolving name for %s", url));
if(titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
}
} catch (XpathException e) {
throw new HostException(e);
}
imageFileData.setImageUrl(url.replace("show", "images"));
imageFileData.setImageName(title.substring(8));
if(title == null || title.isEmpty()) {
title = getDefaultImageName(url);
}
}
@Override
protected void setImageRequest(ImageFileData imageFileData) throws IOException {
HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
try {
imageFileData.setImageUrl(url.replace("show", "images"));
imageFileData.setImageName(title.substring(8));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/host/ImgboxHost.java
+19 -42
View File
@@ -1,26 +1,23 @@
package tn.mnlr.vripper.host;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import java.io.IOException;
@Service
public class ImgboxHost extends Host {
String host = "imgbox.com";
boolean https = true;
private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
private static final String host = "imgbox.com";
public static final String IMG_XPATH = "//img[@id='img']";
@Autowired
private ConnectionManager cm;
@@ -33,45 +30,25 @@ public class ImgboxHost extends Host {
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
String basePage;
Document doc = getDocument(url);
HttpClient client = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(url);
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
basePage = EntityUtils.toString(response.getEntity());
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
Document doc = null;
Node imgNode;
try {
doc = htmlProcessorService.clean(basePage);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
Node imgNode = null;
try {
imgNode = xpathService.getAsNode(doc, "//img[@id='img']");
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
try {
logger.info(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(8));
}
@Override
protected void setImageRequest(ImageFileData imageFileData) throws IOException {
HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(8));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java
+30 -42
View File
@@ -4,10 +4,11 @@ import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -25,8 +26,12 @@ import java.util.List;
@Service
public class ImxHost extends Host {
String host = "imx.to";
boolean https = true;
private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
private static final String host = "imx.to";
public static final String CONTINUE_BUTTON_XPATH = "//div[@id='continuetoimage']";
public static final String IMG_XPATH = "//img[@class='centred']";
@Autowired
private ConnectionManager cm;
@@ -38,35 +43,20 @@ public class ImxHost extends Host {
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
String basePage;
Document doc = getDocument(url);
HttpClient client = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(url);
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
basePage = EntityUtils.toString(response.getEntity());
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
Document doc = null;
Node contDiv;
try {
doc = htmlProcessorService.clean(basePage);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
Node contDiv = null;
try {
contDiv = xpathService.getAsNode(doc, "//div[@id='continuetoimage']");
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
e.printStackTrace();
throw new HostException(e);
}
if (contDiv != null) {
client = this.cm.getClient().build();
HttpPost httpPost = this.cm.buildHttpPost(url);
logger.info(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
httpPost.addHeader("Referer", url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to image ... "));
@@ -75,8 +65,9 @@ public class ImxHost extends Host {
} catch (Exception e) {
throw new HostException(e);
}
logger.info(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
logger.info(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
@@ -84,26 +75,23 @@ public class ImxHost extends Host {
}
}
Node imgNode = null;
Node imgNode;
try {
imgNode = xpathService.getAsNode(doc, "//img[@class='centred']");
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
try {
logger.info(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
}
@Override
protected void setImageRequest(ImageFileData imageFileData) throws IOException {
HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
} catch(Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/host/PixhostHost.java
+20 -42
View File
@@ -1,26 +1,24 @@
package tn.mnlr.vripper.host;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import java.io.IOException;
@Service
public class PixhostHost extends Host {
String host = "pixhost.to";
boolean https = false;
private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
private static final String host = "pixhost.to";
public static final String IMG_XPATH = "//img[@id='image']";
@Autowired
private ConnectionManager cm;
@@ -32,45 +30,25 @@ public class PixhostHost extends Host {
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
String basePage;
Document doc = getDocument(url);
HttpClient client = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(url);
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
basePage = EntityUtils.toString(response.getEntity());
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
Document doc = null;
Node imgNode;
try {
doc = htmlProcessorService.clean(basePage);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
Node imgNode = null;
try {
imgNode = xpathService.getAsNode(doc, "//img[@id='image']");
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
try {
logger.info(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1));
}
@Override
protected void setImageRequest(ImageFileData imageFileData) throws IOException {
HttpGet httpGet = this.cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadJob.java
+9 -9
View File
@@ -10,13 +10,13 @@ import java.util.concurrent.Callable;
public class DownloadJob implements Callable<Image> {
private Logger logger = LoggerFactory.getLogger(DownloadJob.class);
private static final Logger logger = LoggerFactory.getLogger(DownloadJob.class);
@Getter
final Image image;
private final Image image;
@Getter
final ImageFileData imageFileData = new ImageFileData();
private final ImageFileData imageFileData = new ImageFileData();
public DownloadJob(Image image) {
this.image = image;
@@ -25,12 +25,12 @@ public class DownloadJob implements Callable<Image> {
@Override
public Image call() throws Exception {
logger.debug(String.format("Starting downloading %s", image.getUrl()));
this.image.setStatus(Image.Status.DOWNLOADING);
this.image.setCurrent(0);
this.image.getHost().download(this.image, this.imageFileData);
this.image.setStatus(Image.Status.COMPLETE);
return this.image;
logger.info(String.format("Starting downloading %s", image.getUrl()));
image.setStatus(Image.Status.DOWNLOADING);
image.setCurrent(0);
image.getHost().download(image, imageFileData);
image.setStatus(Image.Status.COMPLETE);
return image;
}
@Override
vripper-server/src/main/java/tn/mnlr/vripper/q/DownloadQ.java
+15 -15
View File
@@ -18,7 +18,7 @@ import java.util.stream.Collectors;
@Service
public class DownloadQ {
private Logger logger = LoggerFactory.getLogger(DownloadQ.class);
private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
@Autowired
private AppStateService appStateService;
@@ -33,18 +33,18 @@ public class DownloadQ {
public synchronized void put(Image image) {
try {
logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
DownloadJob downloadJob = new DownloadJob(image);
this.downloadQ.put(downloadJob);
this.appStateService.newDownloadJob(downloadJob);
downloadQ.put(downloadJob);
appStateService.newDownloadJob(downloadJob);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
public DownloadJob take() throws InterruptedException {
DownloadJob downloadJob = this.downloadQ.take();
logger.debug(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
DownloadJob downloadJob = downloadQ.take();
logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
return downloadJob;
}
@@ -68,7 +68,7 @@ public class DownloadQ {
return;
}
appStateService.getPost(postId).setStatus(Post.Status.PENDING);
logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
images.forEach(image -> {
image.init();
put(image);
@@ -77,16 +77,16 @@ public class DownloadQ {
public void removeScheduled(Image image) {
image.setStatus(Image.Status.STOPPED);
logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
Iterator<DownloadJob> iterator = this.downloadQ.iterator();
Iterator<DownloadJob> iterator = downloadQ.iterator();
boolean removed = false;
while(iterator.hasNext()) {
DownloadJob next = iterator.next();
if(next.getImage().getPostId().equals(image.getPostId())) {
iterator.remove();
appStateService.doneDownloadJob(image);
logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
removed = true;
break;
}
@@ -98,14 +98,14 @@ public class DownloadQ {
}
public void removeRunning(String postId) {
logger.debug(String.format("Interrupting running jobs for post id %s", postId));
this.executionService.stop(postId);
logger.info(String.format("Interrupting running jobs for post id %s", postId));
executionService.stop(postId);
}
public synchronized void stop(String postId) {
try {
this.notPauseQ = false;
notPauseQ = false;
appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
List<Image> images = appStateService.getPost(postId)
.getImages()
@@ -115,11 +115,11 @@ public class DownloadQ {
if (images.isEmpty()) {
return;
}
logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
images.forEach(image -> removeScheduled(image));
removeRunning(postId);
} finally {
this.notPauseQ = true;
notPauseQ = true;
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/q/ExecutionService.java
+9 -9
View File
@@ -29,7 +29,7 @@ import java.util.stream.Collectors;
@Service
public class ExecutionService {
Logger logger = LoggerFactory.getLogger(ExecutionService.class);
private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
@Autowired
private DownloadQ downloadQ;
@@ -70,12 +70,12 @@ public class ExecutionService {
}
public void stop(String postId) {
List<DownloadJob> data = this.running
List<DownloadJob> data = running
.stream()
.filter(e -> e.getImage().getPostId().equals(postId))
.peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
.collect(Collectors.toList());
logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
data.forEach(e -> {
futures.get(e.getImage().getUrl()).cancel(true);
@@ -85,7 +85,7 @@ public class ExecutionService {
boolean canRun() {
boolean canRun = threadCount.get() < settings.getMaxThreads();
if (canRun && this.downloadQ.isNotPauseQ()) {
if (canRun && downloadQ.isNotPauseQ()) {
threadCount.incrementAndGet();
return true;
}
@@ -97,7 +97,7 @@ public class ExecutionService {
if (canRun()) {
DownloadJob take = null;
try {
take = this.downloadQ.take();
take = downloadQ.take();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -109,15 +109,15 @@ public class ExecutionService {
Failsafe.with(retryPolicy)
.onFailure(e -> {
if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
logger.debug("Job successfully interrupted");
logger.info("Job successfully interrupted");
return;
}
logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
finalTake.getImage().setStatus(Image.Status.ERROR);
})
.onComplete(e -> {
this.appStateService.doneDownloadJob(finalTake.getImage());
logger.debug(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
appStateService.doneDownloadJob(finalTake.getImage());
logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
synchronized (threadCount) {
int i = threadCount.decrementAndGet();
running.remove(finalTake);
@@ -127,7 +127,7 @@ public class ExecutionService {
})
.get(finalTake::call);
};
logger.debug(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
futures.put(finalTake.getImage().getUrl(), executor.submit(task));
} else {
synchronized (threadCount) {
vripper-server/src/main/java/tn/mnlr/vripper/q/ImageFileData.java
-2
View File
@@ -2,7 +2,6 @@ package tn.mnlr.vripper.q;
import lombok.Getter;
import lombok.Setter;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
@Getter
@@ -12,6 +11,5 @@ public class ImageFileData {
private String pageUrl;
private String imageName;
private String imageUrl;
// private CloseableHttpResponse imageResponse;
private HttpUriRequest imageRequest;
}
vripper-server/src/main/java/tn/mnlr/vripper/services/AppStateService.java
+10 -10
View File
@@ -38,7 +38,7 @@ public class AppStateService {
public void onImageUpdate(Image imageState) {
this.persistenceService.getProcessor().onNext(currentPosts);
persistenceService.getProcessor().onNext(currentPosts);
liveImageUpdates.onNext(imageState);
if (imageState.isCompleted()) {
@@ -57,24 +57,24 @@ public class AppStateService {
}
public Post getPost(String postId) {
return this.currentPosts.get(postId);
return currentPosts.get(postId);
}
public synchronized void newDownloadJob(DownloadJob downloadJob) {
String postId = downloadJob.getImage().getPostId();
checkKeyRunningPosts(postId);
int i = this.runningPosts.get(postId).incrementAndGet();
int i = runningPosts.get(postId).incrementAndGet();
if (i > 0) {
Post post = this.currentPosts.get(postId);
Post post = currentPosts.get(postId);
post.setStatus(Post.Status.DOWNLOADING);
this.livePostsState.onNext(post);
livePostsState.onNext(post);
}
}
public void doneDownloadJob(Image image) {
String postId = image.getPostId();
int i = this.runningPosts.get(postId).decrementAndGet();
Post post = this.currentPosts.get(postId);
int i = runningPosts.get(postId).decrementAndGet();
Post post = currentPosts.get(postId);
if(image.getStatus().equals(Image.Status.ERROR)) {
post.setStatus(Post.Status.PARTIAL);
}
@@ -84,13 +84,13 @@ public class AppStateService {
} else {
post.setStatus(Post.Status.COMPLETE);
}
this.livePostsState.onNext(post);
livePostsState.onNext(post);
}
}
private synchronized void checkKeyRunningPosts(String key) {
if (!this.runningPosts.containsKey(key)) {
this.runningPosts.put(key, new AtomicInteger(0));
if (!runningPosts.containsKey(key)) {
runningPosts.put(key, new AtomicInteger(0));
}
}
}
vripper-server/src/main/java/tn/mnlr/vripper/services/ConnectionManager.java
+1 -2
View File
@@ -10,12 +10,11 @@ import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.stereotype.Service;
@Service
public class ConnectionManager {
private ConnectionManager() {
this.buildConnectionPool();
buildConnectionPool();
}
private PoolingHttpClientConnectionManager pcm;
vripper-server/src/main/java/tn/mnlr/vripper/services/PersistenceService.java
+5 -5
View File
@@ -7,7 +7,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.Main;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit;
@Service
public class PersistenceService {
private Logger logger = LoggerFactory.getLogger(PersistenceService.class);
private static final Logger logger = LoggerFactory.getLogger(PersistenceService.class);
@Autowired
private AppStateService stateService;
@@ -48,7 +48,7 @@ public class PersistenceService {
}
public void persist(Map<String, Post> currentPosts) {
try(PrintWriter out = new PrintWriter(Main.dataPath)) {
try(PrintWriter out = new PrintWriter(VripperApplication.dataPath)) {
out.print(om.writeValueAsString(currentPosts));
} catch (IOException e) {
logger.error("Failed to persist app state", e);
@@ -70,13 +70,13 @@ public class PersistenceService {
String jsonContent;
try {
jsonContent = new String(Files.readAllBytes(Paths.get(Main.dataPath)));
jsonContent = new String(Files.readAllBytes(Paths.get(VripperApplication.dataPath)));
} catch (Exception e) {
logger.warn("data file not found, previous state cannot be restored", e);
return;
}
Map<String, Post> read = this.read(jsonContent);
Map<String, Post> read = read(jsonContent);
stateService.getCurrentPosts().clear();
stateService.getCurrentPosts().putAll(read);
vripper-server/src/main/java/tn/mnlr/vripper/services/PostParser.java
+25 -24
View File
@@ -23,13 +23,14 @@ import java.util.List;
@Service
public class PostParser {
private static final Logger logger = LoggerFactory.getLogger(PostParser.class);
private final static String VIPER_GIRLS_BASE_ADRESS = "https://vipergirls.to/";
private final static String POSTS_XPATH = "//li[contains(@id,'post_')][not(contains(@id,'post_thank'))]";
private final static String REAL_THREAD_XPATH = ".//a[@class='postcounter']";
private final static String THREAD_TITLE_XPATH = "//li[contains(@class, 'lastnavbit')]/span";
private final static String POST_TITLE_XPATH = ".//h2";
private final static String POST_LINKS_XPATH = ".//a";
private Logger logger = LoggerFactory.getLogger(PostParser.class);
@Autowired
private ConnectionManager cm;
@@ -48,14 +49,14 @@ public class PostParser {
public List<Post> parse(String threadUrl) throws PostParseException {
logger.debug(String.format("Parsing thread %s", threadUrl));
logger.info(String.format("Parsing thread %s", threadUrl));
List<Post> posts = new ArrayList<>();
String postResponse;
HttpClient connection = this.cm.getClient().build();
HttpGet httpGet = this.cm.buildHttpGet(threadUrl);
HttpClient connection = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(threadUrl);
logger.debug(String.format("Requesting %s", httpGet));
logger.info(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
@@ -69,7 +70,7 @@ public class PostParser {
Document document;
try {
logger.debug(String.format("Cleaning HTML response for XML parsing: %s", postResponse));
logger.info(String.format("Cleaning HTML response for XML parsing: %s", postResponse));
document = htmlProcessorService.clean(postResponse);
} catch (Exception e) {
throw new PostParseException(e);
@@ -77,11 +78,11 @@ public class PostParser {
String threadTitle = null;
try {
logger.debug(String.format("Looking for thread title using xpath: %s", THREAD_TITLE_XPATH));
logger.info(String.format("Looking for thread title using xpath: %s", THREAD_TITLE_XPATH));
Node threadTitleNode = xpathService.getAsNode(document, THREAD_TITLE_XPATH);
if (threadTitleNode != null) {
threadTitle = threadTitleNode.getTextContent().trim();
logger.debug(String.format("Thread title found %s", threadTitle));
logger.info(String.format("Thread title found %s", threadTitle));
}
if (threadTitle == null) {
logger.warn("Cannot find thread's title");
@@ -92,33 +93,33 @@ public class PostParser {
NodeList postsNodeList;
try {
logger.debug(String.format("Looking for posts using xpath: %s", POSTS_XPATH));
logger.info(String.format("Looking for posts using xpath: %s", POSTS_XPATH));
postsNodeList = xpathService.getAsNodeList(document, POSTS_XPATH);
} catch (Exception e) {
throw new PostParseException(e);
}
logger.debug(String.format("Found %d posts in thread %s", postsNodeList.getLength(), threadUrl));
logger.info(String.format("Found %d posts in thread %s", postsNodeList.getLength(), threadUrl));
for (int i = 0; i < postsNodeList.getLength(); i++) {
String realUrl;
logger.debug(String.format("Parsing post #%d", i + 1));
logger.info(String.format("Parsing post #%d", i + 1));
try {
logger.debug(String.format("Finding posts's link"));
logger.info(String.format("Finding posts's link"));
realUrl = VIPER_GIRLS_BASE_ADRESS.concat(xpathService
.getAsNode(postsNodeList.item(i), REAL_THREAD_XPATH)
.getAttributes()
.getNamedItem("href")
.getTextContent()
.trim());
logger.debug(String.format("Post's link: %s", realUrl));
logger.info(String.format("Post's link: %s", realUrl));
} catch (Exception e) {
throw new PostParseException(e);
}
logger.debug("Finding posts's id");
logger.info("Finding posts's id");
String postId = realUrl.substring(realUrl.indexOf("#")).replace("#post", "");
logger.debug(String.format("Post's id: %s", postId));
logger.info(String.format("Post's id: %s", postId));
if (appStateService.getCurrentPosts().containsKey(postId)) {
logger.warn(String.format("Post with id %s is already loaded, skipping", postId));
@@ -127,18 +128,18 @@ public class PostParser {
String postTitle;
try {
logger.debug(String.format("Finding post's title"));
logger.info(String.format("Finding post's title"));
Node titleNode = xpathService.getAsNode(postsNodeList.item(i), POST_TITLE_XPATH);
if (titleNode != null) {
postTitle = titleNode.getTextContent().trim();
logger.debug(String.format("Found post's title: %s", postTitle));
logger.info(String.format("Found post's title: %s", postTitle));
} else {
logger.debug("Cannot find post's title");
logger.info("Cannot find post's title");
if (threadTitle != null) {
logger.debug("Falling back to thread title to generate a post name");
logger.info("Falling back to thread title to generate a post name");
postTitle = threadTitle + "#" + postId;
} else {
logger.debug("Falling back to post id to generate a post name");
logger.info("Falling back to post id to generate a post name");
postTitle = "#" + postId;
}
}
@@ -148,7 +149,7 @@ public class PostParser {
ArrayList<Image> imagesList = new ArrayList<>();
try {
logger.debug(String.format("Finding all links for post with id %s using xpath %s", postId, POST_LINKS_XPATH));
logger.info(String.format("Finding all links for post with id %s using xpath %s", postId, POST_LINKS_XPATH));
NodeList imagesNodeList = xpathService.getAsNodeList(postsNodeList.item(i), POST_LINKS_XPATH);
for (int j = 0; j < imagesNodeList.getLength(); j++) {
@@ -156,14 +157,14 @@ public class PostParser {
Host foundHost;
if (imageHref != null) {
String imageUrl = imageHref.getTextContent().trim();
logger.debug(String.format("Scanning %s", imageUrl));
logger.info(String.format("Scanning %s", imageUrl));
foundHost = supportedHosts.stream().filter(host -> host.isSupported(imageUrl)).findFirst().orElse(null);
} else {
logger.warn("href is null, skipping");
continue;
}
if (foundHost != null) {
logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref));
logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), imageHref));
imagesList.add(appStateService.createImage(imageHref.getTextContent(), postId, postTitle, foundHost));
} else {
logger.warn(String.format("unsupported host for %s, skipping", imageHref));
@@ -175,7 +176,7 @@ public class PostParser {
}
if (!imagesList.isEmpty()) {
logger.debug(String.format("Found %d images for post with id %s", imagesList.size(), postId));
logger.info(String.format("Found %d images for post with id %s", imagesList.size(), postId));
posts.add(appStateService.createPost(postTitle, realUrl, imagesList, null, postId));
} else {
logger.warn(String.format("No images found for post with id %s, skipping", postId));
vripper-server/src/main/java/tn/mnlr/vripper/services/VipergirlsAuthService.java
+5 -5
View File
@@ -23,7 +23,7 @@ import java.util.List;
@Service
public class VipergirlsAuthService {
private Logger logger = LoggerFactory.getLogger(VipergirlsAuthService.class);
private static final Logger logger = LoggerFactory.getLogger(VipergirlsAuthService.class);
@Autowired
private ConnectionManager cm;
@@ -44,11 +44,11 @@ public class VipergirlsAuthService {
public void authenticate() throws VripperException {
logger.info("Authenticating using ViperGirls credentials");
this.authenticated = false;
authenticated = false;
if(!appSettings.isVLogin()) {
logger.warn("Authentication option is disabled");
this.cookies = null;
cookies = null;
return;
}
@@ -102,7 +102,7 @@ public class VipergirlsAuthService {
} catch (Exception e) {
throw new VripperException(e);
}
this.authenticated = true;
authenticated = true;
logger.info(String.format("Authenticated: %s", username));
}
@@ -143,7 +143,7 @@ public class VipergirlsAuthService {
String postPage = EntityUtils.toString(response.getEntity());
Document document = htmlProcessorService.clean(postPage);
String thanksUrl = this.xpathService
String thanksUrl = xpathService
.getAsNode(document, "//li[contains(@id,'post_')][not(contains(@id,'post_thank'))]//a[@class='post_thanks_button']")
.getAttributes()
.getNamedItem("href")
vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java
+16 -15
View File
@@ -21,7 +21,8 @@ import java.util.List;
@CrossOrigin(value = "*")
public class PostRestEndpoint {
Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
@Autowired
private AppStateService appStateService;
@@ -48,14 +49,14 @@ public class PostRestEndpoint {
@PostMapping("/post")
@ResponseStatus(value = HttpStatus.OK)
public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception {
this.logger.info(String.format("Starting to process thread: %s", url.url));
logger.info(String.format("Starting to process thread: %s", url.url));
if (url.url == null || url.url.isEmpty()) {
return new ResponseEntity("Failed to process empty request", HttpStatus.BAD_REQUEST);
return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST);
} else if (!url.url.startsWith("https://vipergirls.to")) {
return new ResponseEntity("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
}
List<Post> parsed = this.postParser.parse(url.url);
this.logger.debug(String.format("%d posts found from thread %s", parsed.size(), url.url));
List<Post> parsed = postParser.parse(url.url);
logger.info(String.format("%d posts found from thread %s", parsed.size(), url.url));
parsed.forEach(p -> {
try {
authService.leaveThanks(p.getUrl(), p.getPostId());
@@ -63,28 +64,28 @@ public class PostRestEndpoint {
logger.error(String.format("Failed to leave thanks for %s", p.getUrl()), e);
}
});
if(this.appSettings.isAutoStart()) {
this.logger.info("Auto start downloads option is enabled");
this.logger.debug(String.format("Starting to enqueue %d jobs for %s", parsed.stream().flatMap(e -> e.getImages().stream()).count(), url.url));
parsed.forEach(post -> this.downloadQ.enqueue(post));
this.logger.debug(String.format("Done enqueuing jobs for %s", url.url));
if(appSettings.isAutoStart()) {
logger.info("Auto start downloads option is enabled");
logger.info(String.format("Starting to enqueue %d jobs for %s", parsed.stream().flatMap(e -> e.getImages().stream()).count(), url.url));
parsed.forEach(post -> downloadQ.enqueue(post));
logger.info(String.format("Done enqueuing jobs for %s", url.url));
} else {
this.logger.info("Auto start downloads option is disabled");
logger.info("Auto start downloads option is disabled");
}
this.logger.info(String.format("Done processing thread: %s", url.url));
logger.info(String.format("Done processing thread: %s", url.url));
return new ResponseEntity(HttpStatus.OK);
}
@PostMapping("/post/restart")
@ResponseStatus(value = HttpStatus.OK)
public void restartPost(@RequestBody PostId postId) throws Exception {
this.downloadQ.restart(postId.getPostId());
downloadQ.restart(postId.getPostId());
}
@PostMapping("/post/stop")
@ResponseStatus(value = HttpStatus.OK)
public void stop(@RequestBody PostId postId) throws Exception {
this.downloadQ.stop(postId.getPostId());
downloadQ.stop(postId.getPostId());
}
@Getter
vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java
+8 -8
View File
@@ -15,7 +15,7 @@ import tn.mnlr.vripper.services.VipergirlsAuthService;
@CrossOrigin(value = "*")
public class SettingsRestEndpoint {
private Logger logger = LoggerFactory.getLogger(SettingsRestEndpoint.class);
private static final Logger logger = LoggerFactory.getLogger(SettingsRestEndpoint.class);
@Autowired
private AppSettings settings;
@@ -68,13 +68,13 @@ public class SettingsRestEndpoint {
public AppSettings.Settings getSettings() throws Exception {
return new AppSettings.Settings(
this.settings.getDownloadPath(),
this.settings.getMaxThreads(),
this.settings.isAutoStart(),
this.settings.isVLogin(),
this.settings.getVUsername(),
this.settings.getVPassword(),
this.settings.isVThanks()
settings.getDownloadPath(),
settings.getMaxThreads(),
settings.isAutoStart(),
settings.isVLogin(),
settings.getVUsername(),
settings.getVPassword(),
settings.isVThanks()
);
}
vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/WebSocketHandler.java
+7 -7
View File
@@ -29,7 +29,7 @@ import java.util.stream.Collectors;
@Component
public class WebSocketHandler extends TextWebSocketHandler {
private Logger logger = LoggerFactory.getLogger(WebSocketHandler.class);
private static final Logger logger = LoggerFactory.getLogger(WebSocketHandler.class);
@Getter
private static class WSMessage {
@@ -69,11 +69,11 @@ public class WebSocketHandler extends TextWebSocketHandler {
subscribeForPostDetails(session, wsMessage.getPayload());
break;
case POST_DETAILS_UNSUB:
logger.debug("unsubscribe from post details");
logger.info(String.format("Client %s unsubscribed from post details", session.getId()));
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
break;
case POSTS_UNSUB:
logger.debug("unsubscribe from posts");
logger.info(String.format("Client %s unsubscribed from posts", session.getId()));
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
break;
}
@@ -81,7 +81,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
private void subscribeForPosts(WebSocketSession session) {
logger.debug(String.format("Client %s subscribed for posts", session.getId()));
logger.info(String.format("Client %s subscribed for posts", session.getId()));
if (postsSubscriptions.containsKey(session.getId())) {
postsSubscriptions.get(session.getId()).dispose();
}
@@ -93,13 +93,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
.map(e -> e.stream().distinct().collect(Collectors.toList()))
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> this.send(session, msg), e -> logger.error("Failed to send data to client", e))
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private void subscribeForPostDetails(WebSocketSession session, String postId) {
logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
logger.info(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
if (postDetailsSubscriptions.containsKey(session.getId())) {
postDetailsSubscriptions.get(session.getId()).dispose();
}
@@ -112,7 +112,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
.map(e -> e.stream().distinct().collect(Collectors.toList()))
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> this.send(session, msg), e -> logger.error("Failed to send data to client", e))
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
vripper-server/src/main/resources/application.properties
+3 -3
View File
@@ -1,3 +1,3 @@
logging.level.org.springframework.web=DEBUG
logging.level.root=DEBUG
logging.level.org.apache.http=WARN
logging.level.org.springframework.web=ERROR
logging.level.root=ERROR
logging.level.org.apache.http=ERROR