Compare commits

..
19 Commits
Author SHA1 Message Date
dev-claw a8e5dbf5f7 change package id 2025-09-29 17:50:02 +01:00
dev-claw 70cee5cb17 change artifact id 2025-09-29 17:46:48 +01:00
dev-claw 9012cc065e backport fixes for Imx and ImageBam 2025-09-29 17:37:50 +01:00
death-claw c018033d2f Code enhancements 2021-05-31 20:22:16 +01:00
death-claw b0f5a15943 Add support for dpic.me 2021-05-31 19:06:41 +01:00
death-claw 8db4d406ad v3.5.4 2021-05-29 10:32:26 +01:00
death-claw 47f1705abb Fix download queue bug 2021-05-29 10:30:18 +01:00
death-claw 816fb84549 v3.5.3 2021-05-24 15:17:30 +01:00
death-claw 5213ed7129 Fix icon 2021-05-24 15:15:49 +01:00
death-claw 2483c5fa00 v3.5.2 2021-05-24 14:59:32 +01:00
death-claw dfc5810fd3 Add missing icon 2021-05-24 14:57:47 +01:00
death-claw fad418134e Update donation infos 2021-05-24 14:56:00 +01:00
death-claw 9dd91bfc07 v3.5.1 2021-05-22 15:51:05 +01:00
death-claw 5bd42e2ce5 Merge remote-tracking branch 'origin/master' 2021-05-22 15:44:22 +01:00
death-claw e18832b154 v3.5.0 2021-05-21 20:11:16 +01:00
death-claw 41feb53a83 Bug fix 2021-05-20 23:46:59 +01:00
death-claw 6f166b42c4 Enhance metadata interruption
Add Added on and Order columns
2021-05-20 21:25:52 +01:00
death-claw e8ee579a37 Save window size and position
Resize confirmation box
2021-05-18 21:10:42 +01:00
death-claw 41ce99995b Download service enhancements 2021-05-10 21:19:40 +01:00
131 changed files with 1939 additions and 1634 deletions
+28
View File
@@ -1,5 +1,33 @@
# Changelog
## [3.5.4] - 2021-05-29
### Changed
- Fix download queue bug
## [3.5.3] - 2021-05-24
### Changed
- Update donation infos
## [3.5.1] - 2021-05-22
### Changed
- Bug fix
## [3.5.0] - 2021-05-21
### Changed
- Download queue bug fix
- Save window size and position
- Resize confirmation box
- Enhance metadata interruption
- Add Added on and Order columns
## [3.4.2] - 2021-05-10
### Changed
+12 -7
View File
@@ -1,16 +1,20 @@
# VRipper!
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website.
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website. The purpose of
this project is to build a robust and clean application to conveniently download photo galleries using modern web
technologies, Java + Spring boot for the back end and angular + electron for the front end.
## How to build
You need a recent version of maven 3.6.1+.
The application support 2 build modes:
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there
for you.
To build the server app:
@@ -25,6 +29,7 @@ version of nodejs on your system.
If you feel generous and want to support me, you can donate some coins, and I would be very grateful 🙏
I accept Bitcoin (BTC) for donations, following is my address
I accept [PayPal](https://www.paypal.com/myaccount/transfer/homepage/buy/preview) donations, following is my email
address
bc1q9jzzw3st49r9vmxgtylq4vrd67cl7zfclf3lsq
dev.vripper@gmail.com
+4 -4
View File
@@ -2,14 +2,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.4.2</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.4.5</version>
<version>3.5.6</version>
<relativePath/>
</parent>
<modules>
+13 -3
View File
@@ -7,6 +7,7 @@ const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const windowStateKeeper = require('electron-window-state');
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
@@ -29,7 +30,7 @@ process.on("uncaughtException", err => {
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId('tn.mnlr.vripper');
app.setAppUserModelId('me.vripper.vripper-app');
}
let icon;
@@ -38,9 +39,17 @@ createWindow = () => {
} else if (process.platform === 'linux') {
icon = __dirname + '/icon.png';
}
const mainWindowState = windowStateKeeper({
defaultWidth: 1024,
defaultHeight: 800
});
win = new BrowserWindow({
width: 1024,
height: 800,
x: mainWindowState.x,
y: mainWindowState.y,
width: mainWindowState.width,
height: mainWindowState.height,
minWidth: 800,
minHeight: 600,
webPreferences: {
@@ -50,6 +59,7 @@ createWindow = () => {
icon: icon
});
mainWindowState.manage(win);
win.removeMenu();
win.setMenu(null);
win.setMenuBarVisibility(false);
+5 -4
View File
@@ -1,13 +1,13 @@
{
"name": "vripper-electron",
"version": "3.4.2",
"version": "3.6.0",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
"homepage": "https://github.com/death-claw/vripper-project",
"author": "dev-claw",
"homepage": "https://github.com/dev-claw/vripper-project",
"license": "ISC",
"build": {
"appId": "tn.mnlr.vripper",
"appId": "me.vripper.vripper-app",
"productName": "VRipper",
"files": [
"**/*",
@@ -76,6 +76,7 @@
"cheerio": "1.0.0-rc.3",
"copy-dir": "1.3.0",
"electron-context-menu": "2.1.0",
"electron-window-state": "^5.0.3",
"get-port": "5.1.1",
"rimraf": "3.0.2",
"v8-compile-cache": "2.1.1"
+5 -4
View File
@@ -4,12 +4,13 @@
xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.4.2</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<packaging>pom</packaging>
<build>
<plugins>
<plugin>
@@ -28,7 +29,7 @@
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<groupId>me.vripper</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
-10
View File
@@ -1,15 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="web" name="Web">
<configuration>
<webroots />
</configuration>
</facet>
<facet type="Spring" name="Spring">
<configuration />
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
+380 -360
View File
File diff suppressed because it is too large Load Diff
+25 -4
View File
@@ -3,9 +3,9 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.4.2</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
@@ -55,6 +55,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.14</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
@@ -75,18 +76,33 @@
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>tn.mnlr</groupId>
<groupId>me.vripper</groupId>
<artifactId>vripper-ui</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<type>pom</type>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.14.0</version>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.4.2</version>
<configuration>
<classifier>${buildClassifier}</classifier>
<archive>
@@ -112,6 +128,11 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
</plugins>
</build>
@@ -1,11 +1,12 @@
package tn.mnlr.vripper;
package me.vripper;
import lombok.Getter;
import me.vripper.jpa.repositories.IPostRepository;
import me.vripper.services.DataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
@Component
public class EventListenerBean {
@@ -13,15 +14,18 @@ public class EventListenerBean {
@Getter private static boolean init = false;
private final IPostRepository postRepository;
private final DataService dataService;
@Autowired
public EventListenerBean(IPostRepository postRepository) {
public EventListenerBean(IPostRepository postRepository, DataService dataService) {
this.postRepository = postRepository;
this.dataService = dataService;
}
@EventListener
public void onApplicationEvent(ContextRefreshedEvent event) {
postRepository.setDownloadingToStopped();
dataService.sortPostsByRank();
init = true;
}
}
@@ -1,5 +1,6 @@
package tn.mnlr.vripper;
package me.vripper;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
@@ -7,8 +8,6 @@ import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.stereotype.Component;
import java.util.Map;
@Component
@Slf4j
public class SpringContext implements ApplicationContextAware {
@@ -1,10 +1,9 @@
package tn.mnlr.vripper;
import lombok.extern.slf4j.Slf4j;
package me.vripper;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Utils {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper;
package me.vripper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
@@ -1,7 +1,25 @@
package tn.mnlr.vripper.download;
package me.vripper.download;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.services.*;
import me.vripper.services.domain.Settings;
import net.jodah.failsafe.function.CheckedRunnable;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
@@ -10,25 +28,6 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.domain.Settings;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
@Slf4j
public class DownloadJob implements CheckedRunnable {
@@ -45,7 +44,7 @@ public class DownloadJob implements CheckedRunnable {
@Getter private final Image image;
@Getter private final Post post;
private volatile boolean stopped = false;
@Getter private boolean finished = false;
@Getter private volatile boolean finished = false;
DownloadJob(Post post, Image image, Settings settings) {
this.image = image;
@@ -98,7 +97,7 @@ public class DownloadJob implements CheckedRunnable {
log.debug(
String.format(
"Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image, context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
log.debug(
String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
@@ -1,16 +1,15 @@
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.jpa.domain.LogEvent;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.DataService;
package me.vripper.download;
import java.time.LocalDateTime;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.ConnectionService;
import me.vripper.services.DataService;
import net.jodah.failsafe.Failsafe;
@Slf4j
public class DownloadJobWrapper implements Runnable {
@@ -1,63 +1,83 @@
package tn.mnlr.vripper.download;
package me.vripper.download;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
import jakarta.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.services.DataService;
import me.vripper.services.MetadataService;
import me.vripper.services.SettingsService;
import me.vripper.services.domain.Settings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class DownloadService {
private static final List<Status> FINISHED =
Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
private final int MAX_POOL_SIZE = 12;
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final SettingsService settingsService;
private final DataService dataService;
private final MetadataService metadataService;
private final List<Host> hosts;
private final Map<Host, AtomicInteger> threadCount = new HashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final List<DownloadJob> running = new ArrayList<>();
private final List<DownloadJob> pending = new ArrayList<>();
private final SettingsService settingsService;
private final DataService dataService;
private final PostService postService;
private final List<Host> hosts;
private boolean pauseQ = false;
private Thread pollThread;
private final Thread pollThread;
@Autowired
public DownloadService(
SettingsService settingsService,
DataService dataService,
PostService postService,
MetadataService metadataService,
List<Host> hosts) {
this.settingsService = settingsService;
this.dataService = dataService;
this.postService = postService;
this.metadataService = metadataService;
this.hosts = hosts;
pollThread =
new Thread(
() -> {
while (!Thread.interrupted()) {
try {
synchronized (this) {
List<DownloadJob> accepted = new ArrayList<>();
List<DownloadJob> candidates = getCandidates(candidateCount());
candidates.forEach(
c -> {
if (canRun(c.getImage().getHost())) {
accepted.add(c);
}
});
pending.removeAll(accepted);
accepted.forEach(this::schedule);
accepted.clear();
this.wait();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
},
"Download scheduler thread");
}
@PostConstruct
private void init() {
pollThread = new Thread(this::start, "Polling thread");
pollThread.start();
}
@@ -72,7 +92,10 @@ public class DownloadService {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
executor.awaitTermination(5, TimeUnit.SECONDS);
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
log.warn("Some jobs are still running!, forcing shutdown");
executor.shutdownNow();
}
}
private synchronized void stopRunning(@NonNull String postId) {
@@ -90,43 +113,50 @@ public class DownloadService {
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(500);
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::stop);
} else {
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
}
public void stopAll(List<String> postIds) {
stop(
Objects.requireNonNullElseGet(
postIds,
() ->
dataService.findAllPosts().stream()
.map(Post::getPostId)
.collect(Collectors.toList())));
}
public void restartAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::restart);
} else {
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
}
restart(
Objects.requireNonNullElseGet(
posIds,
() ->
dataService.findAllPosts().stream()
.map(Post::getPostId)
.collect(Collectors.toList())));
}
private void restart(@NonNull String postId) {
if (isPending(postId)) {
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
private void restart(@NonNull List<String> postIds) {
Map<Post, Collection<Image>> data = new HashMap<>();
for (String postId : postIds) {
if (isPending(postId)) {
log.warn(
String.format("Cannot restart, jobs are currently running for post id %s", postIds));
continue;
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
continue;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postIds));
data.put(post, images);
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
return;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
post.setStatus(Status.PENDING);
dataService.updatePostStatus(post.getStatus(), post.getId());
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
enqueue(post, images);
enqueue(data);
}
private synchronized boolean isPending(String postId) {
@@ -137,24 +167,19 @@ public class DownloadService {
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private synchronized void stop(String postId) {
try {
pauseQ = true;
private synchronized void stop(List<String> postIds) {
for (String postId : postIds) {
final Post post = dataService.findPostByPostId(postId).orElseThrow();
if (post == null) {
return;
}
if (FINISHED.contains(post.getStatus())) {
return;
continue;
}
pending.removeIf(p -> p.getPost().equals(post));
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
postService.stopFetchingMetadata(post);
} finally {
pauseQ = false;
}
metadataService.stopFetchingMetadata(postIds);
}
private boolean canRun(Host host) {
@@ -165,7 +190,7 @@ public class DownloadService {
&& (settingsService.getSettings().getMaxTotalThreads() == 0
? totalRunning < MAX_POOL_SIZE
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
if (canRun && !pauseQ) {
if (canRun) {
threadCount.get(host).incrementAndGet();
return true;
}
@@ -186,66 +211,41 @@ public class DownloadService {
return map;
}
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount, int max) {
int i = 0;
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount) {
List<DownloadJob> candidates = new ArrayList<>();
for (DownloadJob downloadJob : pending) {
Host host = downloadJob.getImage().getHost();
Integer maxPerHost = candidateCount.get(host);
if (maxPerHost > 0 && i < max) {
if (maxPerHost > 0) {
candidates.add(downloadJob);
candidateCount.put(host, maxPerHost + 1);
i++;
}
if (i >= max) {
break;
candidateCount.put(host, maxPerHost - 1);
}
}
return candidates;
}
public void enqueue(Post post, Collection<Image> imageList) {
public void enqueue(Map<Post, Collection<Image>> images) {
synchronized (this) {
for (Image image : imageList) {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob =
new DownloadJob(post, image, (Settings) settingsService.getSettings().clone());
pending.add(downloadJob);
for (Map.Entry<Post, Collection<Image>> entry : images.entrySet()) {
entry.getKey().setStatus(Status.PENDING);
dataService.updatePostStatus(entry.getKey().getStatus(), entry.getKey().getId());
for (Image image : entry.getValue()) {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob =
new DownloadJob(
entry.getKey(), image, (Settings) settingsService.getSettings().clone());
pending.add(downloadJob);
}
}
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
this.notify();
}
}
private void start() {
while (!Thread.interrupted()) {
try {
synchronized (this) {
List<DownloadJob> accepted = new ArrayList<>();
List<DownloadJob> candidates = getCandidates(candidateCount(), MAX_POOL_SIZE);
candidates.forEach(
c -> {
if (canRun(c.getImage().getHost())) {
accepted.add(c);
}
});
pending.removeAll(accepted);
accepted.forEach(this::push);
accepted.clear();
this.wait();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
private synchronized void push(DownloadJob downloadJob) {
private synchronized void schedule(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new DownloadJobWrapper(downloadJob));
running.add(downloadJob);
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.event;
package me.vripper.event;
import lombok.Getter;
@@ -1,11 +1,10 @@
package tn.mnlr.vripper.event;
package me.vripper.event;
import jakarta.annotation.PreDestroy;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import javax.annotation.PreDestroy;
@Service
public class EventBus {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class DownloadException extends Exception {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class HostException extends Exception {
public HostException(Throwable e) {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class HtmlProcessorException extends Exception {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class PostParseException extends Exception {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class QueueException extends Exception {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
import java.io.IOException;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class ValidationException extends Exception {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class VripperException extends Exception {
public VripperException(String message) {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.exception;
package me.vripper.exception;
public class XpathException extends Exception {
public XpathException(Throwable e) {
@@ -1,6 +1,15 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.ConnectionService;
import me.vripper.services.HostService;
import me.vripper.services.HtmlProcessorService;
import me.vripper.services.XpathService;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -13,15 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
@@ -59,24 +59,26 @@ public class AcidimgHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node contDiv;
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(
String.format(
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, image.getUrl()));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
log.debug(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", image.getUrl()));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url, context);
httpPost.addHeader("Referer", url);
HttpPost httpPost = cm.buildHttpPost(image.getUrl(), context);
httpPost.addHeader("Referer", image.getUrl());
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
try {
@@ -98,7 +100,7 @@ public class AcidimgHost extends Host {
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
@@ -109,7 +111,7 @@ public class AcidimgHost extends Host {
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -0,0 +1,86 @@
package me.vripper.host;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class DPicMeHost extends Host {
private static final String IMG_XPATH = "//img[@id='pic']";
private static final String host = "dpic.me";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public DPicMeHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
HostService.Response response =
hostService.getResponse(image.getUrl().replace("http://", "https://"), context);
Document doc = response.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException(
String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.getUrl()));
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(e -> e.getTextContent().trim())
.orElse("");
String imgUrl =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("src"))
.map(e -> e.getTextContent().trim())
.orElse("");
String defaultName = UUID.randomUUID().toString();
int index = imgUrl.lastIndexOf('/');
if (index != -1 && index < imgUrl.length()) {
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
}
return new HostService.NameUrl(imgTitle.isEmpty() ? defaultName : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,12 +1,12 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.services.HostService;
package me.vripper.host;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@@ -21,7 +21,7 @@ public abstract class Host {
}
public abstract HostService.NameUrl getNameAndUrl(
final String url, final HttpClientContext context) throws HostException;
final Image image, final HttpClientContext context) throws HostException;
@Override
public boolean equals(Object o) {
@@ -1,18 +1,20 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
import java.util.UUID;
@Service
@Slf4j
@@ -42,17 +44,22 @@ public class ImageBamHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
context.setCookieStore(new BasicCookieStore());
BasicClientCookie cookie = new BasicClientCookie("nsfw_inter", "1");
cookie.setDomain("www.imagebam.com");
context.getCookieStore().addCookie(cookie);
HostService.Response response = hostService.getResponse(url, context);
HostService.Response response = hostService.getResponse(image.getUrl(), context);
Document doc = response.getDocument();
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, url));
log.debug(
String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, image.getUrl()));
if (xpathService.getAsNode(doc, CONTINUE_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
response = hostService.getResponse(url, context);
response = hostService.getResponse(image.getUrl(), context);
doc = response.getDocument();
}
} catch (XpathException e) {
@@ -61,18 +68,19 @@ public class ImageBamHost extends Host {
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException(String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, url));
throw new HostException(
String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.getUrl()));
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(e -> e.getTextContent().trim())
@@ -1,17 +1,17 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
@@ -40,21 +40,21 @@ public class ImageTwistHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(Node::getTextContent)
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -39,10 +40,10 @@ public class ImageVenueHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = _url.replace("http://", "https://");
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -39,16 +40,16 @@ public class ImageZillaHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
String title;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
log.debug(String.format("Resolving name for %s", url));
log.debug(String.format("Resolving name for %s", image.getUrl()));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
@@ -59,11 +60,11 @@ public class ImageZillaHost extends Host {
}
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(url);
title = hostService.getDefaultImageName(image.getUrl());
}
try {
return new HostService.NameUrl(title, url.replace("show", "images"));
return new HostService.NameUrl(title, image.getUrl().replace("show", "images"));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -38,10 +39,10 @@ public class ImgSpiceHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = _url.replace("http://", "https://");
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -38,21 +39,21 @@ public class ImgboxHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -0,0 +1,40 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class ImxHost extends Host {
private static final String host = "imx.to";
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = String.format("IMG_%04d", image.getIndex() + 1);
String imgUrl =
image
.getThumbUrl()
.replace("http:", "https:")
.replace("upload/small/", "u/i/")
.replace("u/t/", "u/i/");
return new HostService.NameUrl(imgTitle, imgUrl);
}
}
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -39,12 +40,12 @@ public class PimpandhostHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url;
try {
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
url = hostService.appendUri(image.getUrl().replace("http://", "https://"), "size=original");
} catch (Exception e) {
throw new HostException(e);
}
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -38,10 +39,10 @@ public class PixRouteHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = _url.replace("http://", "https://");
String url = image.getUrl().replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -39,21 +40,21 @@ public class PixhostHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -39,10 +40,10 @@ public class PixxxelsHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = _url.replace("http://", "https://");
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
@@ -1,17 +1,17 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
@@ -41,10 +41,10 @@ public class PostImgHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = _url.replace("http://", "https://");
String url = image.getUrl().replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node urlNode, titleNode;
@@ -1,15 +1,16 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -39,16 +40,17 @@ public class TurboImageHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
String title;
try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
log.debug(
String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.getUrl()));
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Resolving name for %s", url));
log.debug(String.format("Resolving name for %s", image.getUrl()));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
@@ -59,7 +61,7 @@ public class TurboImageHost extends Host {
}
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(url);
title = hostService.getDefaultImageName(image.getUrl());
}
try {
@@ -1,17 +1,17 @@
package tn.mnlr.vripper.host;
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
@@ -40,21 +40,21 @@ public class ViprImHost extends Host {
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(Node::getTextContent)
@@ -1,15 +1,7 @@
package tn.mnlr.vripper.jpa;
package me.vripper.jpa;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.services.ThreadPoolService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -17,6 +9,13 @@ import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import me.vripper.download.DownloadService;
import me.vripper.services.ThreadPoolService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Component
@EnableScheduling
@@ -0,0 +1,24 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
public class DateTimeSerializer extends StdSerializer<LocalDateTime> {
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
protected DateTimeSerializer() {
super(LocalDateTime.class);
}
@Override
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
throws IOException {
gen.writeString(value.format(DATE_TIME_FORMATTER));
}
}
@@ -1,14 +1,13 @@
package tn.mnlr.vripper.jpa.domain;
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Objects;
import me.vripper.host.Host;
import me.vripper.jpa.domain.enums.Status;
@Getter
@Setter
@@ -22,6 +21,8 @@ public class Image {
private String url;
private String thumbUrl;
private int index;
private long current = 0;
@@ -34,9 +35,10 @@ public class Image {
@JsonIgnore private Long postIdRef;
public Image(String postId, String url, Host host, int index) {
public Image(String postId, String url, String thumbUrl, Host host, int index) {
this.postId = postId;
this.url = url;
this.thumbUrl = thumbUrl;
this.host = host;
this.index = index;
status = Status.STOPPED;
@@ -1,18 +1,12 @@
package tn.mnlr.vripper.jpa.domain;
package me.vripper.jpa.domain;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import java.time.LocalDateTime;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@Getter
@Setter
@ToString
@@ -55,19 +49,3 @@ public class LogEvent {
ERROR
}
}
class DateTimeSerializer extends StdSerializer<LocalDateTime> {
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
protected DateTimeSerializer() {
super(LocalDateTime.class);
}
@Override
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
throws IOException {
gen.writeString(value.format(DATE_TIME_FORMATTER));
}
}
@@ -1,11 +1,10 @@
package tn.mnlr.vripper.jpa.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
package me.vripper.jpa.domain;
import java.util.Collections;
import java.util.List;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@@ -1,16 +1,16 @@
package tn.mnlr.vripper.jpa.domain;
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import me.vripper.jpa.domain.enums.Status;
@Getter
@Setter
@@ -50,6 +50,9 @@ public class Post {
private Metadata metadata;
private int rank = Integer.MAX_VALUE;
@JsonSerialize(using = DateTimeSerializer.class)
private LocalDateTime addedOn;
public Post(
@@ -1,13 +1,12 @@
package tn.mnlr.vripper.jpa.domain;
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.util.Objects;
@Getter
@Setter
@NoArgsConstructor
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.jpa.domain.enums;
package me.vripper.jpa.domain.enums;
public enum Status {
PENDING,
@@ -1,10 +1,9 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.enums.Status;
public interface IImageRepository extends IRepository {
@@ -1,9 +1,8 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.LogEvent;
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.LogEvent;
public interface ILogEventRepository extends IRepository {
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Metadata;
package me.vripper.jpa.repositories;
import java.util.Optional;
import me.vripper.jpa.domain.Metadata;
public interface IMetadataRepository extends IRepository {
@@ -1,10 +1,9 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
public interface IPostRepository extends IRepository {
@@ -33,4 +32,6 @@ public interface IPostRepository extends IRepository {
int updateTitle(String title, Long id);
int updateThanked(boolean thanked, Long id);
int updateRank(int rank, Long id);
}
@@ -1,9 +1,8 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Queued;
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Queued;
public interface IQueuedRepository extends IRepository {
Queued save(Queued queued);
@@ -0,0 +1,3 @@
package me.vripper.jpa.repositories;
public interface IRepository {}
@@ -1,21 +1,20 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.SpringContext;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IImageRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class ImageRepository implements IImageRepository {
@@ -37,7 +36,7 @@ public class ImageRepository implements IImageRepository {
public Image save(Image image) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, THUMB_URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?,?)",
id,
image.getCurrent(),
image.getHost().getHost(),
@@ -46,6 +45,7 @@ public class ImageRepository implements IImageRepository {
image.getStatus().name(),
image.getTotal(),
image.getUrl(),
image.getThumbUrl(),
image.getPostIdRef());
image.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
@@ -145,6 +145,7 @@ class ImageRowMapper implements RowMapper<Image> {
.findAny()
.orElse(null));
image.setUrl(rs.getString("URL"));
image.setThumbUrl(rs.getString("THUMB_URL"));
image.setIndex(rs.getInt("INDEX"));
image.setCurrent(rs.getLong("CURRENT"));
image.setTotal(rs.getLong("TOTAL"));
@@ -1,15 +1,4 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.LogEvent;
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
import tn.mnlr.vripper.services.SettingsService;
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -17,6 +6,16 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.SettingsService;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@@ -1,18 +1,17 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.repositories.IMetadataRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class MetadataRepository implements IMetadataRepository {
@@ -1,15 +1,4 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -17,6 +6,16 @@ import java.sql.Timestamp;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IPostRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class PostRepository implements IPostRepository {
@@ -38,7 +37,7 @@ public class PostRepository implements IPostRepository {
public Post save(Post post) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON, RANK) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
@@ -54,7 +53,8 @@ public class PostRepository implements IPostRepository {
post.getTitle(),
post.getTotal(),
post.getUrl(),
Timestamp.valueOf(post.getAddedOn()));
Timestamp.valueOf(post.getAddedOn()),
post.getRank());
post.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return post;
@@ -172,6 +172,14 @@ public class PostRepository implements IPostRepository {
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateRank(int rank, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.RANK = ? WHERE post.ID = ?", rank, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
}
class PostRowMapper implements RowMapper<Post> {
@@ -201,6 +209,7 @@ class PostRowMapper implements RowMapper<Post> {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
post.setRank(rs.getInt("post.RANK"));
Long metadataId = rs.getLong("metadata.POST_ID_REF");
if (!rs.wasNull()) {
@@ -1,18 +1,17 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.IQueuedRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class QueuedRepository implements IQueuedRepository {
@@ -1,7 +1,16 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import jakarta.annotation.PreDestroy;
import java.net.URI;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.download.DownloadJob;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.services.domain.Settings;
import net.jodah.failsafe.RetryPolicy;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
@@ -17,16 +26,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
import tn.mnlr.vripper.download.DownloadJob;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PreDestroy;
import java.net.URI;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Service
@EnableScheduling
@@ -1,21 +1,19 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import java.util.*;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.domain.*;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IImageRepository;
import me.vripper.jpa.repositories.IMetadataRepository;
import me.vripper.jpa.repositories.IPostRepository;
import me.vripper.jpa.repositories.IQueuedRepository;
import me.vripper.jpa.repositories.impl.LogEventRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tn.mnlr.vripper.jpa.domain.*;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
import tn.mnlr.vripper.jpa.repositories.impl.LogEventRepository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@Service
@Transactional
@@ -68,6 +66,7 @@ public class DataService {
image.setPostIdRef(post.getId());
save(image);
});
sortPostsByRank();
}
public synchronized void afterJobFinish(Image image, Post post) {
@@ -100,6 +99,10 @@ public class DataService {
postRepository.updateFolderName(postFolderName, id);
}
public void updatePostRank(int rank, Long id) {
postRepository.updateRank(rank, id);
}
public void finishPost(@NonNull Post post) {
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
post.setStatus(Status.ERROR);
@@ -112,16 +115,19 @@ public class DataService {
post.setStatus(Status.COMPLETE);
updatePostStatus(post.getStatus(), post.getId());
if (settingsService.getSettings().getClearCompleted()) {
remove(post.getPostId());
remove(List.of(post.getPostId()));
}
}
}
}
private void remove(@NonNull final String postId) {
imageRepository.deleteAllByPostId(postId);
metadataRepository.deleteByPostId(postId);
postRepository.deleteByPostId(postId);
private void remove(@NonNull final List<String> postIds) {
for (String postId : postIds) {
imageRepository.deleteAllByPostId(postId);
metadataRepository.deleteByPostId(postId);
postRepository.deleteByPostId(postId);
}
sortPostsByRank();
}
public void newQueueLink(@NonNull final Queued queued) {
@@ -134,18 +140,16 @@ public class DataService {
public List<String> clearCompleted() {
List<String> completed = postRepository.findCompleted();
completed.forEach(this::remove);
remove(completed);
return completed;
}
public void removeAll(final List<String> postIds) {
if (postIds != null && !postIds.isEmpty()) {
for (String postId : postIds) {
remove(postId);
}
} else {
postRepository.findAll().forEach(p -> remove(p.getPostId()));
}
remove(
Objects.requireNonNullElse(
postIds,
postRepository.findAll().stream().map(Post::getPostId).collect(Collectors.toList())));
}
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
@@ -226,4 +230,13 @@ public class DataService {
public void clearQueueLinks() {
queuedRepository.deleteAll();
}
public synchronized void sortPostsByRank() {
List<Post> posts = findAllPosts();
posts.sort(Comparator.comparing(Post::getAddedOn));
for (int i = 0; i < posts.size(); i++) {
posts.get(i).setRank(i);
updatePostRank(i, posts.get(i).getId());
}
}
}
@@ -1,13 +1,12 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import java.util.concurrent.atomic.AtomicLong;
import lombok.Getter;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import java.util.concurrent.atomic.AtomicLong;
@Service
@EnableScheduling
@@ -1,14 +1,14 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import lombok.Getter;
import me.vripper.download.DownloadService;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.services.domain.GlobalState;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.services.domain.GlobalState;
@Service
@EnableScheduling
@@ -1,7 +1,12 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.HtmlProcessorException;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
@@ -11,12 +16,6 @@ import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@Service
@Slf4j
@@ -1,12 +1,12 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import me.vripper.exception.HtmlProcessorException;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.exception.HtmlProcessorException;
@Service
public class HtmlProcessorService {
@@ -0,0 +1,49 @@
package me.vripper.services;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.domain.Post;
import me.vripper.tasks.MetadataRunnable;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class MetadataService {
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
private final ThreadPoolService threadPoolService;
public MetadataService(ThreadPoolService threadPoolService) {
this.threadPoolService = threadPoolService;
}
public void startFetchingMetadata(Post post) {
MetadataRunnable runnable = new MetadataRunnable(post);
threadPoolService.getGeneralExecutor().submit(runnable);
fetchingMetadata.put(post.getPostId(), runnable);
}
public void stopFetchingMetadata(List<String> postIds) {
List<MetadataRunnable> stopping = new ArrayList<>();
for (Map.Entry<String, MetadataRunnable> entry : this.fetchingMetadata.entrySet()) {
if (postIds.contains(entry.getKey())) {
stopping.add(entry.getValue());
entry.getValue().stop();
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(MetadataRunnable::isFinished);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
postIds.forEach(fetchingMetadata::remove);
}
}
@@ -1,18 +1,17 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.RenameException;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.domain.Settings;
package me.vripper.services;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.concurrent.locks.ReentrantLock;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.RenameException;
import me.vripper.jpa.domain.Post;
import me.vripper.services.domain.Settings;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@@ -1,24 +1,22 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.domain.MultiPostScanParser;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
import tn.mnlr.vripper.tasks.AddPostRunnable;
import tn.mnlr.vripper.tasks.AddQueuedRunnable;
import javax.annotation.PreDestroy;
import jakarta.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.domain.MultiPostScanParser;
import me.vripper.services.domain.MultiPostScanResult;
import me.vripper.tasks.AddPostRunnable;
import me.vripper.tasks.AddQueuedRunnable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@Slf4j
@@ -26,19 +24,14 @@ public class PostService {
private final DataService dataService;
private final ThreadPoolService threadPoolService;
private final MetadataService metadataService;
private final LoadingCache<Queued, MultiPostScanResult> cache;
private final Disposable disposable;
@Autowired
public PostService(
DataService dataService,
ThreadPoolService threadPoolService,
MetadataService metadataService,
EventBus eventBus) {
DataService dataService, ThreadPoolService threadPoolService, EventBus eventBus) {
this.dataService = dataService;
this.threadPoolService = threadPoolService;
this.metadataService = metadataService;
cache =
Caffeine.newBuilder()
.expireAfterWrite(5, TimeUnit.MINUTES)
@@ -57,10 +50,6 @@ public class PostService {
}
}
public void stopFetchingMetadata(Post post) {
metadataService.stopFetchingMetadata(post);
}
public void processMultiPost(List<Queued> queuedList) {
for (Queued queued : queuedList) {
if (queued.getPostId() != null) {
@@ -1,25 +1,13 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import static java.nio.file.StandardOpenOption.*;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.ValidationException;
import tn.mnlr.vripper.services.domain.Settings;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
@@ -31,8 +19,19 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static java.nio.file.StandardOpenOption.*;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.exception.ValidationException;
import me.vripper.services.domain.Settings;
import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
@Service
@Setter
@@ -1,11 +1,10 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import org.springframework.stereotype.Service;
package me.vripper.services;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import org.springframework.stereotype.Service;
@Service
public class ThreadPoolService {
@@ -1,7 +1,16 @@
package tn.mnlr.vripper.services;
package me.vripper.services;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.exception.VripperException;
import me.vripper.jpa.domain.Post;
import me.vripper.tasks.LeaveThanksRunnable;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
@@ -15,16 +24,6 @@ import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
import tn.mnlr.vripper.event.Event;
import tn.mnlr.vripper.event.EventBus;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.tasks.LeaveThanksRunnable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
@@ -1,13 +1,12 @@
package tn.mnlr.vripper.services;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import tn.mnlr.vripper.exception.XpathException;
package me.vripper.services;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import me.vripper.exception.XpathException;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@Service
public class XpathService {
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
package me.vripper.services.domain;
import java.util.Objects;
import lombok.Getter;
@Getter
public class DownloadSpeed {
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
package me.vripper.services.domain;
import java.util.Objects;
import lombok.Getter;
@Getter
public class GlobalState {
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
package me.vripper.services.domain;
import java.util.List;
import lombok.Getter;
@Getter
public class MultiPostItem {
@@ -1,15 +1,14 @@
package tn.mnlr.vripper.services.domain;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.SettingsService;
package me.vripper.services.domain;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import me.vripper.SpringContext;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.SettingsService;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
public class MultiPostScanHandler extends DefaultHandler {
@@ -1,24 +1,23 @@
package tn.mnlr.vripper.services.domain;
package me.vripper.services.domain;
import java.io.BufferedInputStream;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.SAXParserFactory;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.ConnectionService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.VGAuthService;
import javax.xml.parsers.SAXParserFactory;
import java.io.BufferedInputStream;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public class MultiPostScanParser {
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
package me.vripper.services.domain;
import java.util.List;
import lombok.Getter;
@Getter
public class MultiPostScanResult {
@@ -1,19 +1,18 @@
package tn.mnlr.vripper.services.domain;
import lombok.extern.slf4j.Slf4j;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.SettingsService;
package me.vripper.services.domain;
import java.util.Collection;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.services.SettingsService;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
@Slf4j
class PostScanHandler extends DefaultHandler {
@@ -72,10 +71,9 @@ class PostScanHandler extends DefaultHandler {
break;
case "image":
index++;
String thumbUrl = attributes.getValue("thumb_url").trim();
if (previews.size() < 4) {
Optional.ofNullable(attributes.getValue("thumb_url"))
.map(String::trim)
.ifPresent(previews::add);
previews.add(thumbUrl);
}
String mainUrl =
@@ -91,7 +89,7 @@ class PostScanHandler extends DefaultHandler {
String.format(
"Found supported host %s for %s",
foundHost.getClass().getSimpleName(), mainUrl));
images.add(new Image(postId, mainUrl, foundHost, index));
images.add(new Image(postId, mainUrl, thumbUrl, foundHost, index));
} else {
log.warn(String.format("unsupported host for %s, skipping", mainUrl));
}
@@ -1,22 +1,21 @@
package tn.mnlr.vripper.services.domain;
package me.vripper.services.domain;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.SAXParserFactory;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.services.ConnectionService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.VGAuthService;
import javax.xml.parsers.SAXParserFactory;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public class PostScanParser {
@@ -1,11 +1,10 @@
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
package me.vripper.services.domain;
import java.util.Optional;
import java.util.Set;
import lombok.Getter;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
public class PostScanResult {
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.domain;
package me.vripper.services.domain;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -1,27 +1,27 @@
package tn.mnlr.vripper.tasks;
package me.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.LogEvent;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.MetadataService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.VGAuthService;
import tn.mnlr.vripper.services.domain.PostScanParser;
import tn.mnlr.vripper.services.domain.PostScanResult;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import static me.vripper.jpa.domain.LogEvent.Status.PROCESSING;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.Set;
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.PROCESSING;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.download.DownloadService;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.DataService;
import me.vripper.services.MetadataService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import me.vripper.services.domain.PostScanParser;
import me.vripper.services.domain.PostScanResult;
@Slf4j
public class AddPostRunnable implements Runnable {
@@ -106,13 +106,12 @@ public class AddPostRunnable implements Runnable {
Set<Image> images = postScanResult.getImages();
dataService.newPost(post, images);
metadataService.startFetchingMetadata(post);
if (settingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
downloadService.enqueue(post, images);
downloadService.enqueue(Map.of(post, images));
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
@@ -1,20 +1,19 @@
package tn.mnlr.vripper.tasks;
package me.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.jpa.domain.LogEvent;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.ThreadPoolService;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import static me.vripper.jpa.domain.LogEvent.Status.PROCESSING;
import java.time.LocalDateTime;
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.PROCESSING;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.DataService;
import me.vripper.services.PostService;
import me.vripper.services.ThreadPoolService;
import me.vripper.services.domain.MultiPostScanResult;
@Slf4j
public class AddQueuedRunnable implements Runnable {
@@ -1,6 +1,18 @@
package tn.mnlr.vripper.tasks;
package me.vripper.tasks;
import java.io.UnsupportedEncodingException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.ConnectionService;
import me.vripper.services.DataService;
import me.vripper.services.SettingsService;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
@@ -9,19 +21,6 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.jpa.domain.LogEvent;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.SettingsService;
import java.io.UnsupportedEncodingException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@Slf4j
public class LeaveThanksRunnable implements Runnable {
@@ -1,22 +1,21 @@
package tn.mnlr.vripper.tasks;
package me.vripper.tasks;
import lombok.extern.slf4j.Slf4j;
import org.springframework.lang.NonNull;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.jpa.domain.LogEvent;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.repositories.impl.LogEventRepository;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.SettingsService;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.impl.LogEventRepository;
import me.vripper.services.PostService;
import me.vripper.services.SettingsService;
import org.springframework.lang.NonNull;
@Slf4j
public class LinkScanRunnable implements Runnable {
@@ -0,0 +1,245 @@
package me.vripper.tasks;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.download.DownloadJob;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.*;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Slf4j
public class MetadataRunnable implements Runnable {
private static final List<String> dictionary =
Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
@Getter private final Post post;
private final DataService dataService;
private final ILogEventRepository eventRepository;
private final LogEvent logEvent;
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
private final XpathService xpathService;
private final HttpClientContext context = HttpClientContext.create();
private final MetadataService metadataService;
private volatile boolean stopped = false;
@Getter private volatile boolean finished = false;
public MetadataRunnable(@NonNull Post post) {
this.post = post;
dataService = SpringContext.getBean(DataService.class);
eventRepository = SpringContext.getBean(ILogEventRepository.class);
cm = SpringContext.getBean(ConnectionService.class);
VGAuthService vgAuthService = SpringContext.getBean(VGAuthService.class);
htmlProcessorService = SpringContext.getBean(HtmlProcessorService.class);
xpathService = SpringContext.getBean(XpathService.class);
metadataService = SpringContext.getBean(MetadataService.class);
context.setCookieStore(vgAuthService.getContext().getCookieStore());
context.setAttribute(
DownloadJob.ContextAttributes.OPEN_CONNECTION.toString(),
Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
logEvent =
new LogEvent(
LogEvent.Type.METADATA,
LogEvent.Status.PENDING,
LocalDateTime.now(),
"Fetching metadata for " + post.getUrl());
eventRepository.save(logEvent);
}
@Override
public void run() {
try {
logEvent.setStatus(LogEvent.Status.PROCESSING);
eventRepository.update(logEvent);
Metadata metadata = fetchMetadata(post.getPostId(), post.getThreadId(), post.getUrl());
if (metadata != null && !stopped) {
dataService.setMetadata(post, metadata);
logEvent.setStatus(LogEvent.Status.DONE);
} else {
logEvent.setStatus(ERROR);
logEvent.setMessage(String.format("Fetching metadata for %s failed", post.getUrl()));
}
eventRepository.update(logEvent);
} catch (Exception e) {
String message = String.format("Failed to fetch metadata for %s", post.getUrl());
log.error(message, e);
logEvent.setMessage(message + "\n" + Utils.throwableToString(e));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
} finally {
if (stopped) {
String message = String.format("Fetching metadata for %s interrupted", post.getUrl());
logEvent.setStatus(LogEvent.Status.DONE);
logEvent.setMessage(message);
eventRepository.update(logEvent);
}
finished = true;
metadataService.stopFetchingMetadata(List.of(post.getPostId()));
}
}
private Metadata fetchMetadata(String postId, String threadId, String url) {
HttpGet httpGet = cm.buildHttpGet(url, context);
AtomicReference<Metadata> metadataReference = new AtomicReference<>();
Failsafe.with(cm.getRetryPolicy())
.onFailure(
e ->
log.error(
String.format("Error occurred when getting post metadata, postId %s", postId),
e.getFailure()))
.run(
() -> {
if (stopped) {
return;
}
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response =
(CloseableHttpResponse) connection.execute(httpGet, context)) {
if (stopped) {
return;
}
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(
String.format(
"Unexpected response code '%d' for %s",
response.getStatusLine().getStatusCode(), httpGet));
}
try {
Document document =
htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
if (stopped) {
return;
}
Node postNode =
xpathService.getAsNode(
document,
String.format(
"//li[@id='post_%s']/div[contains(@class, 'postdetails')]", postId));
if (stopped) {
return;
}
String postedBy =
xpathService
.getAsNode(
postNode,
"./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font")
.getTextContent()
.trim();
if (stopped) {
return;
}
Metadata metadata = new Metadata();
metadata.setPostedBy(postedBy);
if (stopped) {
return;
}
Node node =
xpathService.getAsNode(
document, String.format("//div[@id='post_message_%s']", postId));
metadata.setResolvedNames(findTitleInContent(node));
metadata.setPostId(postId);
if (stopped) {
return;
}
metadataReference.set(metadata);
} catch (Exception e) {
if (stopped) {
log.warn(e.getMessage(), e);
return;
}
throw new PostParseException(
String.format("Failed to parse thread %s, post %s", threadId, postId), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
return metadataReference.get();
}
private List<String> findTitleInContent(Node node) {
List<String> altTitle = new ArrayList<>();
findTitle(node, altTitle, new AtomicBoolean(true));
return altTitle.stream().distinct().collect(Collectors.toList());
}
private void findTitle(Node node, List<String> altTitle, AtomicBoolean keepGoing) {
if (!keepGoing.get()) {
return;
}
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
keepGoing.set(false);
return;
}
if (node.getNodeType() == Node.ELEMENT_NODE) {
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
Node item = node.getChildNodes().item(i);
findTitle(item, altTitle, keepGoing);
if (!keepGoing.get()) {
return;
}
}
} else if (node.getNodeType() == Node.TEXT_NODE) {
String text = node.getTextContent().trim();
if (!text.isBlank()
&& dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
altTitle.add(text);
}
}
}
public void stop() {
this.stopped = true;
List<AbstractExecutionAwareRequest> requests =
(List<AbstractExecutionAwareRequest>)
this.context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
}
}
@@ -1,12 +1,12 @@
package tn.mnlr.vripper.web.restendpoints;
package me.vripper.web.restendpoints;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.repositories.ILogEventRepository;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
@Slf4j
@RestController
@@ -1,29 +1,28 @@
package tn.mnlr.vripper.web.restendpoints;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.domain.MultiPostItem;
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
import tn.mnlr.vripper.tasks.AddPostRunnable;
import tn.mnlr.vripper.tasks.LinkScanRunnable;
import tn.mnlr.vripper.web.restendpoints.domain.*;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
import tn.mnlr.vripper.web.restendpoints.exceptions.ServerErrorException;
package me.vripper.web.restendpoints;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.download.DownloadService;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.*;
import me.vripper.services.domain.MultiPostItem;
import me.vripper.services.domain.MultiPostScanResult;
import me.vripper.tasks.AddPostRunnable;
import me.vripper.tasks.LinkScanRunnable;
import me.vripper.web.restendpoints.domain.*;
import me.vripper.web.restendpoints.exceptions.BadRequestException;
import me.vripper.web.restendpoints.exceptions.NotFoundException;
import me.vripper.web.restendpoints.exceptions.ServerErrorException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
@Slf4j
@RestController
@@ -1,15 +1,14 @@
package tn.mnlr.vripper.web.restendpoints;
package me.vripper.web.restendpoints;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.ValidationException;
import me.vripper.services.SettingsService;
import me.vripper.services.domain.Settings;
import me.vripper.web.restendpoints.exceptions.BadRequestException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import tn.mnlr.vripper.exception.ValidationException;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.domain.Settings;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
import java.util.List;
@RestController
@Slf4j
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints;
package me.vripper.web.restendpoints;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain;
package me.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain;
package me.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain;
package me.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain;
package me.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -1,12 +1,11 @@
package tn.mnlr.vripper.web.restendpoints.domain;
package me.vripper.web.restendpoints.domain;
import java.util.List;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.util.List;
@Getter
@Setter
@NoArgsConstructor

Some files were not shown because too many files have changed in this diff Show More