mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e18832b154 | ||
|
|
6f166b42c4 | ||
|
|
e8ee579a37 | ||
|
|
41ce99995b | ||
|
|
5caeef0e70 | ||
|
|
96ef0cb234 | ||
|
|
641b3e3e7b | ||
|
|
08de4dc5b0 | ||
|
|
57680a6053 | ||
|
|
53fd14935b | ||
|
|
90af03d68f | ||
|
|
265614585b | ||
|
|
4503d540ed | ||
|
|
b08b730dff |
@@ -1,5 +1,25 @@
|
||||
# Changelog
|
||||
|
||||
## [3.5.0] - 2021-05-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Download queue bug fix
|
||||
- Save window size and position
|
||||
- Resize confirmation box
|
||||
- Enhance metadata interruption
|
||||
- Add Added on and Order columns
|
||||
|
||||
## [3.4.2] - 2021-05-10
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix potential issue with imagebam host
|
||||
- Enhance download queue logic
|
||||
- Code source enhancements
|
||||
- Fix app icons
|
||||
- Fix race condition bug
|
||||
|
||||
## [3.3.8] - 2021-05-05
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.8</version>
|
||||
<version>3.5.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
<version>2.4.5</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
<modules>
|
||||
<module>vripper-server</module>
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 2.1 KiB |
@@ -7,6 +7,7 @@ const {spawn} = require("child_process");
|
||||
const {ipcMain} = require("electron");
|
||||
const {dialog} = require("electron");
|
||||
const axios = require('axios');
|
||||
const windowStateKeeper = require('electron-window-state');
|
||||
|
||||
// non null value when it is an AppImage
|
||||
const appImageDir = process.env.APPDIR;
|
||||
@@ -29,18 +30,26 @@ process.on("uncaughtException", err => {
|
||||
createWindow = () => {
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
app.setAppUserModelId("tn.mnlr.vripper");
|
||||
app.setAppUserModelId('tn.mnlr.vripper');
|
||||
}
|
||||
|
||||
let icon;
|
||||
if (process.platform === "win32") {
|
||||
if (process.platform === 'win32') {
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
} else if (process.platform === 'linux') {
|
||||
icon = __dirname + '/icon.png';
|
||||
}
|
||||
|
||||
const mainWindowState = windowStateKeeper({
|
||||
defaultWidth: 1024,
|
||||
defaultHeight: 800
|
||||
});
|
||||
|
||||
win = new BrowserWindow({
|
||||
width: 1024,
|
||||
height: 800,
|
||||
x: mainWindowState.x,
|
||||
y: mainWindowState.y,
|
||||
width: mainWindowState.width,
|
||||
height: mainWindowState.height,
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
webPreferences: {
|
||||
@@ -50,6 +59,7 @@ createWindow = () => {
|
||||
icon: icon
|
||||
});
|
||||
|
||||
mainWindowState.manage(win);
|
||||
win.removeMenu();
|
||||
win.setMenu(null);
|
||||
win.setMenuBarVisibility(false);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-electron",
|
||||
"version": "3.3.8",
|
||||
"version": "3.5.0",
|
||||
"description": "A ripper for vipergirls.to built using web technolgies",
|
||||
"main": "main.js",
|
||||
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
|
||||
@@ -51,7 +51,6 @@
|
||||
"synopsis": "vipergirls.to ripper",
|
||||
"category": "Utility",
|
||||
"packageCategory": "Utility",
|
||||
"icon": "icons",
|
||||
"target": [
|
||||
"AppImage"
|
||||
]
|
||||
@@ -77,6 +76,7 @@
|
||||
"cheerio": "1.0.0-rc.3",
|
||||
"copy-dir": "1.3.0",
|
||||
"electron-context-menu": "2.1.0",
|
||||
"electron-window-state": "^5.0.3",
|
||||
"get-port": "5.1.1",
|
||||
"rimraf": "3.0.2",
|
||||
"v8-compile-cache": "2.1.1"
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.8</version>
|
||||
<version>3.5.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-electron</artifactId>
|
||||
<name>vripper-electron</name>
|
||||
|
||||
@@ -18,7 +18,7 @@ fs.writeFileSync('./build/vripper-ui/index.html', $.html());
|
||||
|
||||
console.log('Building runtime environment');
|
||||
rimraf.sync('java-runtime');
|
||||
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
|
||||
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.security.jgss,java.sql,jdk.unsupported,jdk.crypto.ec --output java-runtime');
|
||||
if (process.platform === 'linux') {
|
||||
console.log('Stripping libjvm.so');
|
||||
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
|
||||
|
||||
+286
-278
File diff suppressed because it is too large
Load Diff
+4
-15
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.8</version>
|
||||
<version>3.5.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<name>vripper-server</name>
|
||||
@@ -16,14 +16,6 @@
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jersey</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-websocket</artifactId>
|
||||
@@ -43,7 +35,6 @@
|
||||
<dependency>
|
||||
<groupId>org.hsqldb</groupId>
|
||||
<artifactId>hsqldb</artifactId>
|
||||
<version>2.5.1</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -64,12 +55,11 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.5.6</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.htmlcleaner</groupId>
|
||||
<artifactId>htmlcleaner</artifactId>
|
||||
<version>2.22</version>
|
||||
<version>2.24</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
@@ -81,9 +71,8 @@
|
||||
<version>2.0.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>28.1-jre</version>
|
||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
||||
<artifactId>caffeine</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
|
||||
@@ -5,28 +5,23 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.event.ContextRefreshedEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tn.mnlr.vripper.jpa.repositories.IRepository;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
import java.util.Set;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
|
||||
@Component
|
||||
public class EventListenerBean {
|
||||
|
||||
@Getter private static boolean init = false;
|
||||
|
||||
private final DataService dataService;
|
||||
private final Set<IRepository> repositorySet;
|
||||
private final IPostRepository postRepository;
|
||||
|
||||
@Autowired
|
||||
public EventListenerBean(DataService dataService, Set<IRepository> repositorySet) {
|
||||
this.dataService = dataService;
|
||||
this.repositorySet = repositorySet;
|
||||
public EventListenerBean(IPostRepository postRepository) {
|
||||
this.postRepository = postRepository;
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void onApplicationEvent(ContextRefreshedEvent event) {
|
||||
dataService.setDownloadingToStopped();
|
||||
postRepository.setDownloadingToStopped();
|
||||
init = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
@@ -39,22 +40,22 @@ public class DownloadJob implements CheckedRunnable {
|
||||
private final ConnectionService cm;
|
||||
private final VGAuthService authService;
|
||||
private final DownloadSpeedService downloadSpeedService;
|
||||
private final SettingsService settingsService;
|
||||
private final Settings settings;
|
||||
private final HttpClientContext context;
|
||||
@Getter private final Image image;
|
||||
@Getter private final Post post;
|
||||
private volatile boolean stopped = false;
|
||||
@Getter private boolean finished = false;
|
||||
@Getter private volatile boolean finished = false;
|
||||
|
||||
DownloadJob(Post post, Image image) {
|
||||
DownloadJob(Post post, Image image, Settings settings) {
|
||||
this.image = image;
|
||||
this.post = post;
|
||||
this.settings = settings;
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
pathService = SpringContext.getBean(PathService.class);
|
||||
cm = SpringContext.getBean(ConnectionService.class);
|
||||
authService = SpringContext.getBean(VGAuthService.class);
|
||||
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
|
||||
settingsService = SpringContext.getBean(SettingsService.class);
|
||||
context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
context.setAttribute(
|
||||
@@ -81,10 +82,9 @@ public class DownloadJob implements CheckedRunnable {
|
||||
// The post may be updated and the download directory might be set by another thread
|
||||
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
|
||||
if (updatedPost.getDownloadDirectory() == null) {
|
||||
pathService.createDefaultPostFolder(updatedPost);
|
||||
pathService.createDefaultPostFolder(updatedPost, settings);
|
||||
}
|
||||
if (settingsService.getSettings().getLeaveThanksOnStart() != null
|
||||
&& settingsService.getSettings().getLeaveThanksOnStart()) {
|
||||
if (settings.getLeaveThanksOnStart() != null && settings.getLeaveThanksOnStart()) {
|
||||
authService.leaveThanks(updatedPost);
|
||||
}
|
||||
}
|
||||
@@ -225,12 +225,11 @@ public class DownloadJob implements CheckedRunnable {
|
||||
}
|
||||
try {
|
||||
pathService.getDirectoryAccess().lock();
|
||||
File downloadDestinationFolder = pathService.calcDownloadDirectory(post);
|
||||
File downloadDestinationFolder = pathService.calcDownloadDirectory(post, settings);
|
||||
File outImage =
|
||||
new File(
|
||||
downloadDestinationFolder,
|
||||
(settingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "")
|
||||
+ imageName);
|
||||
(settings.getForceOrder() ? String.format("%03d_", index) : "") + imageName);
|
||||
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to rename the image", e);
|
||||
@@ -268,6 +267,7 @@ public class DownloadJob implements CheckedRunnable {
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
this.stopped = true;
|
||||
List<AbstractExecutionAwareRequest> requests =
|
||||
(List<AbstractExecutionAwareRequest>)
|
||||
this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
|
||||
@@ -276,7 +276,6 @@ public class DownloadJob implements CheckedRunnable {
|
||||
request.abort();
|
||||
}
|
||||
}
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
public enum ContextAttributes {
|
||||
|
||||
+11
-11
@@ -4,29 +4,29 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Slf4j
|
||||
public class DownloadRunnable implements Runnable {
|
||||
public class DownloadJobWrapper implements Runnable {
|
||||
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final ConnectionService connectionService;
|
||||
private final IEventRepository eventRepository;
|
||||
private final ILogEventRepository eventRepository;
|
||||
|
||||
private final DownloadJob downloadJob;
|
||||
|
||||
public DownloadRunnable(final DownloadJob downloadJob) {
|
||||
public DownloadJobWrapper(final DownloadJob downloadJob) {
|
||||
downloadService = SpringContext.getBean(DownloadService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
connectionService = SpringContext.getBean(ConnectionService.class);
|
||||
eventRepository = SpringContext.getBean(IEventRepository.class);
|
||||
eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
|
||||
this.downloadJob = downloadJob;
|
||||
}
|
||||
@@ -37,16 +37,16 @@ public class DownloadRunnable implements Runnable {
|
||||
.onFailure(
|
||||
e -> {
|
||||
try {
|
||||
Event event =
|
||||
new Event(
|
||||
Event.Type.DOWNLOAD,
|
||||
Event.Status.ERROR,
|
||||
LogEvent logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.DOWNLOAD,
|
||||
LogEvent.Status.ERROR,
|
||||
LocalDateTime.now(),
|
||||
String.format(
|
||||
"Failed to download %s\n %s",
|
||||
downloadJob.getImage().getUrl(),
|
||||
Utils.throwableToString(e.getFailure())));
|
||||
eventRepository.save(event);
|
||||
eventRepository.save(logEvent);
|
||||
} catch (Exception exp) {
|
||||
log.error("Failed to save event", exp);
|
||||
}
|
||||
@@ -9,59 +9,58 @@ import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.MetadataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DownloadService {
|
||||
|
||||
private static final List<Status> FINISHED =
|
||||
Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
|
||||
private final int MAX_POOL_SIZE = 12;
|
||||
|
||||
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
|
||||
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
|
||||
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
|
||||
private final List<DownloadJob> running = new ArrayList<>();
|
||||
private final List<DownloadJob> pending = new ArrayList<>();
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final SettingsService settings;
|
||||
private final SettingsService settingsService;
|
||||
private final DataService dataService;
|
||||
private final PostService postService;
|
||||
private final MetadataService metadataService;
|
||||
|
||||
private final List<Host> hosts;
|
||||
|
||||
private boolean pauseQ = false;
|
||||
private Thread executionThread;
|
||||
private Thread pollThread;
|
||||
|
||||
@Autowired
|
||||
public DownloadService(
|
||||
PendingQueue pendingQueue,
|
||||
SettingsService settings,
|
||||
SettingsService settingsService,
|
||||
DataService dataService,
|
||||
PostService postService) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
this.settings = settings;
|
||||
MetadataService metadataService,
|
||||
List<Host> hosts) {
|
||||
this.settingsService = settingsService;
|
||||
this.dataService = dataService;
|
||||
this.postService = postService;
|
||||
this.metadataService = metadataService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
executionThread = new Thread(this::start, "Executor thread");
|
||||
pollThread = new Thread(this::poll, "Polling thread");
|
||||
pollThread = new Thread(this::start, "Polling thread");
|
||||
pollThread.start();
|
||||
executionThread.start();
|
||||
}
|
||||
|
||||
public void destroy() throws Exception {
|
||||
log.info("Shutting down ExecutionService");
|
||||
executionThread.interrupt();
|
||||
pollThread.interrupt();
|
||||
executor.shutdown();
|
||||
dataService
|
||||
@@ -74,9 +73,9 @@ public class DownloadService {
|
||||
executor.awaitTermination(5, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private void stopRunning(@NonNull String postId) {
|
||||
private synchronized void stopRunning(@NonNull String postId) {
|
||||
List<DownloadJob> stopping = new ArrayList<>();
|
||||
Iterator<DownloadJob> iterator = executing.iterator();
|
||||
Iterator<DownloadJob> iterator = running.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob downloadJob = iterator.next();
|
||||
if (postId.equals(downloadJob.getPost().getPostId())) {
|
||||
@@ -89,148 +88,190 @@ public class DownloadService {
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(DownloadJob::isFinished);
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void stopAll(List<String> posIds) {
|
||||
if (posIds != null) {
|
||||
posIds.forEach(this::stop);
|
||||
} else {
|
||||
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
|
||||
}
|
||||
public void stopAll(List<String> postIds) {
|
||||
stop(
|
||||
Objects.requireNonNullElseGet(
|
||||
postIds,
|
||||
() ->
|
||||
dataService.findAllPosts().stream()
|
||||
.map(Post::getPostId)
|
||||
.collect(Collectors.toList())));
|
||||
}
|
||||
|
||||
public void restartAll(List<String> posIds) {
|
||||
if (posIds != null) {
|
||||
posIds.forEach(this::restart);
|
||||
} else {
|
||||
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
|
||||
}
|
||||
restart(
|
||||
Objects.requireNonNullElseGet(
|
||||
posIds,
|
||||
() ->
|
||||
dataService.findAllPosts().stream()
|
||||
.map(Post::getPostId)
|
||||
.collect(Collectors.toList())));
|
||||
}
|
||||
|
||||
private void restart(@NonNull String postId) {
|
||||
if (isPending(postId)) {
|
||||
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
|
||||
return;
|
||||
}
|
||||
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
|
||||
if (images.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
post.setStatus(Status.PENDING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
for (Image image : images) {
|
||||
try {
|
||||
pendingQueue.put(post, image);
|
||||
} catch (InterruptedException e) {
|
||||
log.warn("Thread was interrupted", e);
|
||||
Thread.currentThread().interrupt();
|
||||
private void restart(@NonNull List<String> postIds) {
|
||||
Map<Post, Collection<Image>> data = new HashMap<>();
|
||||
for (String postId : postIds) {
|
||||
if (isPending(postId)) {
|
||||
log.warn(
|
||||
String.format("Cannot restart, jobs are currently running for post id %s", postIds));
|
||||
continue;
|
||||
}
|
||||
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
|
||||
if (images.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postIds));
|
||||
data.put(post, images);
|
||||
}
|
||||
enqueue(data);
|
||||
}
|
||||
|
||||
private boolean isPending(String postId) {
|
||||
return pendingQueue.isPending(postId);
|
||||
private synchronized boolean isPending(String postId) {
|
||||
return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
|
||||
}
|
||||
|
||||
private void stop(String postId) {
|
||||
try {
|
||||
pauseQ = true;
|
||||
private synchronized boolean isRunning(String postId) {
|
||||
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
|
||||
}
|
||||
|
||||
private synchronized void stop(List<String> postIds) {
|
||||
|
||||
for (String postId : postIds) {
|
||||
final Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (post == null) {
|
||||
return;
|
||||
continue;
|
||||
}
|
||||
if (FINISHED.contains(post.getStatus())) {
|
||||
return;
|
||||
}
|
||||
pendingQueue.stop(post);
|
||||
pending.removeIf(p -> p.getPost().equals(post));
|
||||
stopRunning(postId);
|
||||
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
|
||||
dataService.finishPost(post);
|
||||
postService.stopFetchingMetadata(post);
|
||||
} finally {
|
||||
pauseQ = false;
|
||||
}
|
||||
metadataService.stopFetchingMetadata(postIds);
|
||||
}
|
||||
|
||||
private boolean canRun(Host host) {
|
||||
boolean canRun;
|
||||
AtomicInteger count = threadCount.get(host);
|
||||
if (count == null) {
|
||||
threadCount.put(host, new AtomicInteger(0));
|
||||
}
|
||||
int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
canRun =
|
||||
threadCount.get(host).get() < settings.getSettings().getMaxThreads()
|
||||
&& (settings.getSettings().getMaxTotalThreads() == 0
|
||||
? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE
|
||||
: threadCount.values().stream().mapToInt(AtomicInteger::get).sum()
|
||||
< settings.getSettings().getMaxTotalThreads());
|
||||
if (canRun && !pauseQ) {
|
||||
threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
|
||||
&& (settingsService.getSettings().getMaxTotalThreads() == 0
|
||||
? totalRunning < MAX_POOL_SIZE
|
||||
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
|
||||
if (canRun) {
|
||||
threadCount.get(host).incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void poll() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
List<DownloadJob> peek = pendingQueue.peek();
|
||||
for (DownloadJob downloadJob : peek) {
|
||||
if (canRun(downloadJob.getImage().getHost())) {
|
||||
executionQueue.offer(downloadJob);
|
||||
pendingQueue.remove(downloadJob);
|
||||
private Map<Host, Integer> candidateCount() {
|
||||
HashMap<Host, Integer> map = new HashMap<>();
|
||||
hosts.forEach(
|
||||
h -> {
|
||||
AtomicInteger count = threadCount.get(h);
|
||||
if (count == null) {
|
||||
count = new AtomicInteger(0);
|
||||
threadCount.put(h, count);
|
||||
}
|
||||
}
|
||||
synchronized (threadCount) {
|
||||
threadCount.wait(2_000);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount, int max) {
|
||||
int i = 0;
|
||||
List<DownloadJob> candidates = new ArrayList<>();
|
||||
for (DownloadJob downloadJob : pending) {
|
||||
Host host = downloadJob.getImage().getHost();
|
||||
Integer maxPerHost = candidateCount.get(host);
|
||||
if (maxPerHost > 0 && i < max) {
|
||||
candidates.add(downloadJob);
|
||||
candidateCount.put(host, maxPerHost + 1);
|
||||
i++;
|
||||
}
|
||||
if (i >= max) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
public void enqueue(Map<Post, Collection<Image>> images) {
|
||||
synchronized (this) {
|
||||
for (Map.Entry<Post, Collection<Image>> entry : images.entrySet()) {
|
||||
entry.getKey().setStatus(Status.PENDING);
|
||||
dataService.updatePostStatus(entry.getKey().getStatus(), entry.getKey().getId());
|
||||
for (Image image : entry.getValue()) {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob =
|
||||
new DownloadJob(
|
||||
entry.getKey(), image, (Settings) settingsService.getSettings().clone());
|
||||
pending.add(downloadJob);
|
||||
}
|
||||
}
|
||||
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
private void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
push(executionQueue.take());
|
||||
synchronized (this) {
|
||||
List<DownloadJob> accepted = new ArrayList<>();
|
||||
List<DownloadJob> candidates = getCandidates(candidateCount(), MAX_POOL_SIZE);
|
||||
candidates.forEach(
|
||||
c -> {
|
||||
if (canRun(c.getImage().getHost())) {
|
||||
accepted.add(c);
|
||||
}
|
||||
});
|
||||
pending.removeAll(accepted);
|
||||
accepted.forEach(this::push);
|
||||
accepted.clear();
|
||||
this.wait();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
log.error("Execution Service failed", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void push(DownloadJob downloadJob) {
|
||||
private synchronized void push(DownloadJob downloadJob) {
|
||||
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
|
||||
executor.execute(new DownloadRunnable(downloadJob));
|
||||
executing.add(downloadJob);
|
||||
executor.execute(new DownloadJobWrapper(downloadJob));
|
||||
running.add(downloadJob);
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(DownloadJob downloadJob) {
|
||||
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
|
||||
if (count == 0) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
executing.remove(downloadJob);
|
||||
synchronized (threadCount) {
|
||||
threadCount.notify();
|
||||
public void afterJobFinish(DownloadJob downloadJob) {
|
||||
synchronized (this) {
|
||||
running.remove(downloadJob);
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
if (!isPending(downloadJob.getPost().getPostId())
|
||||
&& !isRunning(downloadJob.getPost().getPostId())) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
public int pendingCount() {
|
||||
return pending.size();
|
||||
}
|
||||
|
||||
public int runningCount() {
|
||||
return executing.size();
|
||||
return running.size();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.BlockingDeque;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PendingQueue {
|
||||
|
||||
private final DataService dataService;
|
||||
private final SettingsService settingsService;
|
||||
private final List<Host> hosts;
|
||||
|
||||
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ =
|
||||
new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
|
||||
|
||||
@Autowired
|
||||
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
|
||||
this.dataService = dataService;
|
||||
this.settingsService = settingsService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
|
||||
}
|
||||
|
||||
public void put(Post post, Image image) throws InterruptedException {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob = new DownloadJob(post, image);
|
||||
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
|
||||
checkKey(post.getPostId());
|
||||
toBeExecuted.get(post.getPostId()).incrementAndGet();
|
||||
}
|
||||
|
||||
private synchronized void checkKey(String postId) {
|
||||
if (!toBeExecuted.containsKey(postId)) {
|
||||
toBeExecuted.put(postId, new AtomicInteger(0));
|
||||
}
|
||||
}
|
||||
|
||||
public void remove(final DownloadJob downloadJob) {
|
||||
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
|
||||
}
|
||||
|
||||
public List<DownloadJob> peek() {
|
||||
List<DownloadJob> downloadJobs = new ArrayList<>();
|
||||
if (hosts.size() == 0) {
|
||||
return downloadJobs;
|
||||
}
|
||||
for (Host host : hosts) {
|
||||
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
|
||||
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
|
||||
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
|
||||
if (downloadJob != null) {
|
||||
downloadJobs.add(downloadJob);
|
||||
}
|
||||
}
|
||||
}
|
||||
return downloadJobs;
|
||||
}
|
||||
|
||||
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
|
||||
for (Image image : images) {
|
||||
put(post, image);
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
}
|
||||
|
||||
public void stop(Post post) {
|
||||
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
|
||||
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
|
||||
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
|
||||
entry.getValue().removeIf(predicate);
|
||||
decrement(post.getPostId());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isPending(String postId) {
|
||||
return toBeExecuted.containsKey(postId);
|
||||
}
|
||||
|
||||
public synchronized int decrement(String postId) {
|
||||
AtomicInteger counter = toBeExecuted.get(postId);
|
||||
if (counter == null) {
|
||||
return 0;
|
||||
}
|
||||
int count = counter.decrementAndGet();
|
||||
if (count == 0) {
|
||||
toBeExecuted.remove(postId);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public class Event<T> {
|
||||
|
||||
private final T data;
|
||||
private final Kind kind;
|
||||
|
||||
private Event(Kind kind, T data) {
|
||||
this.data = data;
|
||||
this.kind = kind;
|
||||
}
|
||||
|
||||
public static <T> Event<T> wrap(Kind kind, T data) {
|
||||
return new Event<>(kind, data);
|
||||
}
|
||||
|
||||
public enum Kind {
|
||||
POST_UPDATE,
|
||||
POST_REMOVE,
|
||||
IMAGE_UPDATE,
|
||||
METADATA_UPDATE,
|
||||
QUEUED_UPDATE,
|
||||
QUEUED_REMOVE,
|
||||
LOG_EVENT_UPDATE,
|
||||
LOG_EVENT_REMOVE,
|
||||
VG_USER,
|
||||
GLOBAL_STATE,
|
||||
BYTES_PER_SECOND,
|
||||
SETTINGS_UPDATE
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Service
|
||||
public class EventBus {
|
||||
|
||||
public static final Sinks.EmitFailureHandler RETRY = (signalType, emitResult) -> true;
|
||||
private final Sinks.Many<Event<?>> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
public void publishEvent(Event<?> event) {
|
||||
sink.emitNext(event, RETRY);
|
||||
}
|
||||
|
||||
public Flux<Event<?>> flux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(RETRY);
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class EventRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public EventRemoveEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class EventUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public EventUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class ImageUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public ImageUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class MetadataUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long postIdRef;
|
||||
|
||||
public MetadataUpdateEvent(Object source, Long postIdRef) {
|
||||
super(source);
|
||||
this.postIdRef = postIdRef;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class PostRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final String postId;
|
||||
|
||||
public PostRemoveEvent(Object source, String postId) {
|
||||
super(source);
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class PostUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public PostUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class QueuedRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final String threadId;
|
||||
|
||||
public QueuedRemoveEvent(Object source, String threadId) {
|
||||
super(source);
|
||||
this.threadId = threadId;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class QueuedUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public QueuedUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ public class ImageBamHost extends Host {
|
||||
|
||||
private static final String host = "imagebam.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class,'main-image')]";
|
||||
private static final String CONTINUE_XPATH = "//*[contains(text(), 'Continue')]";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
@@ -47,6 +48,17 @@ public class ImageBamHost extends Host {
|
||||
HostService.Response response = hostService.getResponse(url, context);
|
||||
Document doc = response.getDocument();
|
||||
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, url));
|
||||
if (xpathService.getAsNode(doc, CONTINUE_XPATH) != null) {
|
||||
// Button detected. No need to actually click it, just make the call again.
|
||||
response = hostService.getResponse(url, context);
|
||||
doc = response.getDocument();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public class DateTimeSerializer extends StdSerializer<LocalDateTime> {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
|
||||
|
||||
protected DateTimeSerializer() {
|
||||
super(LocalDateTime.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
|
||||
throws IOException {
|
||||
gen.writeString(value.format(DATE_TIME_FORMATTER));
|
||||
}
|
||||
}
|
||||
+2
-23
@@ -1,23 +1,18 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class Event {
|
||||
public class LogEvent {
|
||||
|
||||
private Long id;
|
||||
private Type type;
|
||||
@@ -28,7 +23,7 @@ public class Event {
|
||||
|
||||
private String message;
|
||||
|
||||
public Event(Type type, Status status, LocalDateTime time, String message) {
|
||||
public LogEvent(Type type, Status status, LocalDateTime time, String message) {
|
||||
this.type = type;
|
||||
this.status = status;
|
||||
this.time = time;
|
||||
@@ -55,19 +50,3 @@ public class Event {
|
||||
ERROR
|
||||
}
|
||||
}
|
||||
|
||||
class DateTimeSerializer extends StdSerializer<LocalDateTime> {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
|
||||
|
||||
protected DateTimeSerializer() {
|
||||
super(LocalDateTime.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
|
||||
throws IOException {
|
||||
gen.writeString(value.format(DATE_TIME_FORMATTER));
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
@@ -49,6 +51,11 @@ public class Post {
|
||||
|
||||
private Metadata metadata;
|
||||
|
||||
private int rank = Integer.MAX_VALUE;
|
||||
|
||||
@JsonSerialize(using = DateTimeSerializer.class)
|
||||
private LocalDateTime addedOn;
|
||||
|
||||
public Post(
|
||||
String title,
|
||||
String url,
|
||||
@@ -65,6 +72,7 @@ public class Post {
|
||||
this.threadTitle = threadTitle;
|
||||
this.securityToken = securityToken;
|
||||
status = Status.STOPPED;
|
||||
addedOn = LocalDateTime.now();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IEventRepository extends IRepository {
|
||||
|
||||
Event save(Event event);
|
||||
|
||||
Event update(Event event);
|
||||
|
||||
List<Event> findAll();
|
||||
|
||||
Optional<Event> findById(Long id);
|
||||
|
||||
void delete(Long id);
|
||||
|
||||
void deleteAll();
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface ILogEventRepository extends IRepository {
|
||||
|
||||
LogEvent save(LogEvent logEvent);
|
||||
|
||||
LogEvent update(LogEvent logEvent);
|
||||
|
||||
List<LogEvent> findAll();
|
||||
|
||||
Optional<LogEvent> findById(Long id);
|
||||
|
||||
void delete(Long id);
|
||||
|
||||
void deleteAll();
|
||||
}
|
||||
@@ -33,4 +33,6 @@ public interface IPostRepository extends IRepository {
|
||||
int updateTitle(String title, Long id);
|
||||
|
||||
int updateThanked(boolean thanked, Long id);
|
||||
|
||||
int updateRank(int rank, Long id);
|
||||
}
|
||||
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.EventRemoveEvent;
|
||||
import tn.mnlr.vripper.event.EventUpdateEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class EventRepository implements IEventRepository, ApplicationEventPublisherAware {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final SettingsService settingsService;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
public EventRepository(JdbcTemplate jdbcTemplate, SettingsService settingsService) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.settingsService = settingsService;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_EVENT", Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Event save(@NonNull Event event) {
|
||||
|
||||
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
|
||||
|
||||
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
|
||||
if (count > maxRecords) {
|
||||
List<Long> idList =
|
||||
jdbcTemplate.queryForList(
|
||||
"SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
|
||||
idList.forEach(this::delete);
|
||||
}
|
||||
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
|
||||
id,
|
||||
event.getType().name(),
|
||||
event.getStatus().name(),
|
||||
event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
event.getMessage());
|
||||
event.setId(id);
|
||||
applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, id));
|
||||
return event;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Event update(@NonNull Event event) {
|
||||
if (event.getId() == null) {
|
||||
log.warn("Cannot update entity with null id");
|
||||
return event;
|
||||
}
|
||||
|
||||
jdbcTemplate.update(
|
||||
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
|
||||
event.getType().name(),
|
||||
event.getStatus().name(),
|
||||
event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
event.getMessage(),
|
||||
event.getId());
|
||||
applicationEventPublisher.publishEvent(
|
||||
new EventUpdateEvent(EventRepository.class, event.getId()));
|
||||
return event;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Event> findById(Long id) {
|
||||
List<Event> events =
|
||||
jdbcTemplate.query("SELECT * FROM EVENT WHERE ID = ?", new EventRowMapper(), id);
|
||||
if (events.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(events.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Event> findAll() {
|
||||
return jdbcTemplate.query("SELECT * FROM EVENT", new EventRowMapper());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(Long id) {
|
||||
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
|
||||
applicationEventPublisher.publishEvent(new EventRemoveEvent(EventRepository.class, id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM EVENT");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(
|
||||
@NonNull ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public class EventRowMapper implements RowMapper<Event> {
|
||||
|
||||
@Override
|
||||
public Event mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Event event = new Event();
|
||||
event.setId(rs.getLong("ID"));
|
||||
event.setType(Event.Type.valueOf(rs.getString("TYPE")));
|
||||
event.setStatus(Event.Status.valueOf(rs.getString("STATUS")));
|
||||
event.setTime(LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
|
||||
event.setMessage(rs.getString("MESSAGE"));
|
||||
return event;
|
||||
}
|
||||
}
|
||||
+35
-14
@@ -1,28 +1,32 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.ImageUpdateEvent;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware {
|
||||
public class ImageRepository implements IImageRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public ImageRepository(JdbcTemplate jdbcTemplate) {
|
||||
public ImageRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
@@ -44,7 +48,7 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis
|
||||
image.getUrl(),
|
||||
image.getPostIdRef());
|
||||
image.setId(id);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -105,7 +109,7 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?", status.name(), id);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -114,7 +118,7 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?", current, id);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -123,13 +127,30 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?", total, id);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
}
|
||||
|
||||
class ImageRowMapper implements RowMapper<Image> {
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(
|
||||
@NonNull ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Image image = new Image();
|
||||
image.setId(rs.getLong("ID"));
|
||||
String host = rs.getString("HOST");
|
||||
image.setHost(
|
||||
SpringContext.getBeansOfType(Host.class).values().stream()
|
||||
.filter(e -> e.getHost().equals(host))
|
||||
.findAny()
|
||||
.orElse(null));
|
||||
image.setUrl(rs.getString("URL"));
|
||||
image.setIndex(rs.getInt("INDEX"));
|
||||
image.setCurrent(rs.getLong("CURRENT"));
|
||||
image.setTotal(rs.getLong("TOTAL"));
|
||||
image.setStatus(Status.valueOf(rs.getString("STATUS")));
|
||||
image.setPostId(rs.getString("POST_ID"));
|
||||
image.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
return image;
|
||||
}
|
||||
}
|
||||
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class ImageRowMapper implements RowMapper<Image> {
|
||||
|
||||
@Override
|
||||
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Image image = new Image();
|
||||
image.setId(rs.getLong("ID"));
|
||||
String host = rs.getString("HOST");
|
||||
image.setHost(
|
||||
SpringContext.getBeansOfType(Host.class).values().stream()
|
||||
.filter(e -> e.getHost().equals(host))
|
||||
.findAny()
|
||||
.orElse(null));
|
||||
image.setUrl(rs.getString("URL"));
|
||||
image.setIndex(rs.getInt("INDEX"));
|
||||
image.setCurrent(rs.getLong("CURRENT"));
|
||||
image.setTotal(rs.getLong("TOTAL"));
|
||||
image.setStatus(Status.valueOf(rs.getString("STATUS")));
|
||||
image.setPostId(rs.getString("POST_ID"));
|
||||
image.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
return image;
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LogEventRepository implements ILogEventRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final SettingsService settingsService;
|
||||
private final EventBus eventBus;
|
||||
|
||||
public LogEventRepository(
|
||||
JdbcTemplate jdbcTemplate, SettingsService settingsService, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.settingsService = settingsService;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_EVENT", Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized LogEvent save(@NonNull LogEvent logEvent) {
|
||||
|
||||
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
|
||||
|
||||
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
|
||||
if (count > maxRecords) {
|
||||
List<Long> idList =
|
||||
jdbcTemplate.queryForList(
|
||||
"SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
|
||||
idList.forEach(this::delete);
|
||||
}
|
||||
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
|
||||
id,
|
||||
logEvent.getType().name(),
|
||||
logEvent.getStatus().name(),
|
||||
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
logEvent.getMessage());
|
||||
logEvent.setId(id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, id));
|
||||
return logEvent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public LogEvent update(@NonNull LogEvent logEvent) {
|
||||
if (logEvent.getId() == null) {
|
||||
log.warn("Cannot update entity with null id");
|
||||
return logEvent;
|
||||
}
|
||||
|
||||
jdbcTemplate.update(
|
||||
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
|
||||
logEvent.getType().name(),
|
||||
logEvent.getStatus().name(),
|
||||
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
logEvent.getMessage(),
|
||||
logEvent.getId());
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, logEvent.getId()));
|
||||
return logEvent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<LogEvent> findById(Long id) {
|
||||
List<LogEvent> logEvents =
|
||||
jdbcTemplate.query("SELECT * FROM EVENT WHERE ID = ?", new LogEventRowMapper(), id);
|
||||
if (logEvents.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(logEvents.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LogEvent> findAll() {
|
||||
return jdbcTemplate.query("SELECT * FROM EVENT", new LogEventRowMapper());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(Long id) {
|
||||
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_REMOVE, id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM EVENT");
|
||||
}
|
||||
}
|
||||
|
||||
class LogEventRowMapper implements RowMapper<LogEvent> {
|
||||
|
||||
@Override
|
||||
public LogEvent mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
LogEvent logEvent = new LogEvent();
|
||||
logEvent.setId(rs.getLong("ID"));
|
||||
logEvent.setType(LogEvent.Type.valueOf(rs.getString("TYPE")));
|
||||
logEvent.setStatus(LogEvent.Status.valueOf(rs.getString("STATUS")));
|
||||
logEvent.setTime(
|
||||
LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
|
||||
logEvent.setMessage(rs.getString("MESSAGE"));
|
||||
return logEvent;
|
||||
}
|
||||
}
|
||||
+23
-10
@@ -1,26 +1,29 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.MetadataUpdateEvent;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware {
|
||||
public class MetadataRepository implements IMetadataRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public MetadataRepository(JdbcTemplate jdbcTemplate) {
|
||||
public MetadataRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -31,8 +34,7 @@ public class MetadataRepository implements IMetadataRepository, ApplicationEvent
|
||||
metadata.getPostId(),
|
||||
metadata.getPostedBy(),
|
||||
String.join("%sep%", metadata.getResolvedNames()));
|
||||
applicationEventPublisher.publishEvent(
|
||||
new MetadataUpdateEvent(MetadataRepository.class, metadata.getPostIdRef()));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.METADATA_UPDATE, metadata.getPostIdRef()));
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@@ -55,9 +57,20 @@ public class MetadataRepository implements IMetadataRepository, ApplicationEvent
|
||||
return jdbcTemplate.update(
|
||||
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?", postId);
|
||||
}
|
||||
}
|
||||
|
||||
class MetadataRowMapper implements RowMapper<Metadata> {
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
metadata.setPostId(rs.getString("POST_ID"));
|
||||
metadata.setPostedBy(rs.getString("POSTED_BY"));
|
||||
String resolvedNames = rs.getString("RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
public class MetadataRowMapper implements RowMapper<Metadata> {
|
||||
|
||||
@Override
|
||||
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
metadata.setPostId(rs.getString("POST_ID"));
|
||||
metadata.setPostedBy(rs.getString("POSTED_BY"));
|
||||
String resolvedNames = rs.getString("RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
+74
-20
@@ -1,29 +1,33 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.PostRemoveEvent;
|
||||
import tn.mnlr.vripper.event.PostUpdateEvent;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
public class PostRepository implements IPostRepository, ApplicationEventPublisherAware {
|
||||
public class PostRepository implements IPostRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public PostRepository(JdbcTemplate jdbcTemplate) {
|
||||
public PostRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
@@ -34,7 +38,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
public Post save(Post post) {
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON, RANK) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
post.getDone(),
|
||||
post.getForum(),
|
||||
@@ -49,9 +53,11 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
post.getThreadTitle(),
|
||||
post.getTitle(),
|
||||
post.getTotal(),
|
||||
post.getUrl());
|
||||
post.getUrl(),
|
||||
Timestamp.valueOf(post.getAddedOn()),
|
||||
post.getRank());
|
||||
post.setId(id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return post;
|
||||
}
|
||||
|
||||
@@ -119,7 +125,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
public int deleteByPostId(String postId) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("DELETE FROM POST AS post WHERE post.POST_ID = ?", postId);
|
||||
applicationEventPublisher.publishEvent(new PostRemoveEvent(PostRepository.class, postId));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_REMOVE, postId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -128,7 +134,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?", status.name(), id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -136,7 +142,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
public int updateDone(int done, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?", done, id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -147,7 +153,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
|
||||
postFolderName,
|
||||
id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -155,7 +161,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
public int updateTitle(String title, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?", title, id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -164,13 +170,61 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?", thanked, id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(
|
||||
@NonNull ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
public int updateRank(int rank, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("UPDATE POST AS post SET post.RANK = ? WHERE post.ID = ?", rank, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
}
|
||||
|
||||
class PostRowMapper implements RowMapper<Post> {
|
||||
|
||||
private static final String DELIMITER = ";";
|
||||
|
||||
@Override
|
||||
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
|
||||
Post post = new Post();
|
||||
post.setId(rs.getLong("post.ID"));
|
||||
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
|
||||
post.setPostId(rs.getString("post.POST_ID"));
|
||||
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
|
||||
post.setThreadId(rs.getString("post.THREAD_ID"));
|
||||
post.setTitle(rs.getString("post.TITLE"));
|
||||
post.setUrl(rs.getString("post.URL"));
|
||||
post.setDone(rs.getInt("post.DONE"));
|
||||
post.setTotal(rs.getInt("post.TOTAL"));
|
||||
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
|
||||
post.setForum(rs.getString("post.FORUM"));
|
||||
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
|
||||
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
|
||||
post.setThanked(rs.getBoolean("post.THANKED"));
|
||||
String previews;
|
||||
if ((previews = rs.getString("post.PREVIEWS")) != null) {
|
||||
post.setPreviews(Set.of(previews.split(DELIMITER)));
|
||||
}
|
||||
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
|
||||
post.setRank(rs.getInt("post.RANK"));
|
||||
|
||||
Long metadataId = rs.getLong("metadata.POST_ID_REF");
|
||||
if (!rs.wasNull()) {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(metadataId);
|
||||
metadata.setPostId(rs.getString("metadata.POST_ID"));
|
||||
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
|
||||
post.setMetadata(metadata);
|
||||
}
|
||||
|
||||
return post;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class PostRowMapper implements RowMapper<Post> {
|
||||
|
||||
private static final String DELIMITER = ";";
|
||||
|
||||
@Override
|
||||
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
|
||||
Post post = new Post();
|
||||
post.setId(rs.getLong("post.ID"));
|
||||
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
|
||||
post.setPostId(rs.getString("post.POST_ID"));
|
||||
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
|
||||
post.setThreadId(rs.getString("post.THREAD_ID"));
|
||||
post.setTitle(rs.getString("post.TITLE"));
|
||||
post.setUrl(rs.getString("post.URL"));
|
||||
post.setDone(rs.getInt("post.DONE"));
|
||||
post.setTotal(rs.getInt("post.TOTAL"));
|
||||
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
|
||||
post.setForum(rs.getString("post.FORUM"));
|
||||
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
|
||||
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
|
||||
post.setThanked(rs.getBoolean("post.THANKED"));
|
||||
String previews;
|
||||
if ((previews = rs.getString("post.PREVIEWS")) != null) {
|
||||
post.setPreviews(Set.of(previews.split(DELIMITER)));
|
||||
}
|
||||
|
||||
Long metadataId = rs.getLong("metadata.POST_ID_REF");
|
||||
if (!rs.wasNull()) {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(metadataId);
|
||||
metadata.setPostId(rs.getString("metadata.POST_ID"));
|
||||
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
|
||||
post.setMetadata(metadata);
|
||||
}
|
||||
|
||||
return post;
|
||||
}
|
||||
}
|
||||
+26
-16
@@ -1,27 +1,29 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.QueuedRemoveEvent;
|
||||
import tn.mnlr.vripper.event.QueuedUpdateEvent;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware {
|
||||
public class QueuedRepository implements IQueuedRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public QueuedRepository(JdbcTemplate jdbcTemplate) {
|
||||
public QueuedRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
@@ -40,7 +42,7 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
|
||||
queued.getPostId(),
|
||||
queued.getThreadId());
|
||||
queued.setId(id);
|
||||
applicationEventPublisher.publishEvent(new QueuedUpdateEvent(QueuedRepository.class, id));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_UPDATE, id));
|
||||
return queued;
|
||||
}
|
||||
|
||||
@@ -49,8 +51,8 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
|
||||
List<Queued> queuedList =
|
||||
jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
|
||||
new Object[] {threadId},
|
||||
new QueuedRowMapper());
|
||||
new QueuedRowMapper(),
|
||||
threadId);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
@@ -67,9 +69,7 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
|
||||
public Optional<Queued> findById(Long id) {
|
||||
List<Queued> queuedList =
|
||||
jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?",
|
||||
new Object[] {id},
|
||||
new QueuedRowMapper());
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?", new QueuedRowMapper(), id);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
@@ -81,7 +81,7 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
|
||||
public int deleteByThreadId(String threadId) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?", threadId);
|
||||
applicationEventPublisher.publishEvent(new QueuedRemoveEvent(QueuedRepository.class, threadId));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_REMOVE, threadId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@@ -89,9 +89,19 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM QUEUED");
|
||||
}
|
||||
}
|
||||
|
||||
class QueuedRowMapper implements RowMapper<Queued> {
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Queued queued = new Queued();
|
||||
queued.setId(rs.getLong("ID"));
|
||||
queued.setLink(rs.getString("LINK"));
|
||||
queued.setThreadId(rs.getString("THREAD_ID"));
|
||||
queued.setPostId(rs.getString("POST_ID"));
|
||||
queued.setTotal(rs.getInt("TOTAL"));
|
||||
queued.setLoading(rs.getBoolean("LOADING"));
|
||||
return queued;
|
||||
}
|
||||
}
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class QueuedRowMapper implements RowMapper<Queued> {
|
||||
|
||||
@Override
|
||||
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Queued queued = new Queued();
|
||||
queued.setId(rs.getLong("ID"));
|
||||
queued.setLink(rs.getString("LINK"));
|
||||
queued.setThreadId(rs.getString("THREAD_ID"));
|
||||
queued.setPostId(rs.getString("POST_ID"));
|
||||
queued.setTotal(rs.getInt("TOTAL"));
|
||||
queued.setLoading(rs.getBoolean("LOADING"));
|
||||
return queued;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public interface DataEventListener<T> {
|
||||
Flux<T> getDataFlux();
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Sinks.EmitFailureHandler;
|
||||
|
||||
public class EmitHandler {
|
||||
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.EventRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EventRemoveEventListener
|
||||
implements ApplicationListener<EventRemoveEvent>, DataEventListener<EventRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<EventRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(EventRemoveEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<EventRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.EventUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EventUpdateEventListener
|
||||
implements ApplicationListener<EventUpdateEvent>, DataEventListener<EventUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<EventUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(@NonNull EventUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<EventUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.ImageUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class ImageUpdateEventListener
|
||||
implements ApplicationListener<ImageUpdateEvent>, DataEventListener<ImageUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<ImageUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(ImageUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ImageUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.MetadataUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class MetadataUpdateEventListener
|
||||
implements ApplicationListener<MetadataUpdateEvent>, DataEventListener<MetadataUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<MetadataUpdateEvent> sink =
|
||||
Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(MetadataUpdateEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<MetadataUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.PostRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class PostRemoveEventListener
|
||||
implements ApplicationListener<PostRemoveEvent>, DataEventListener<PostRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<PostRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(PostRemoveEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<PostRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.PostUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class PostUpdateEventListener
|
||||
implements ApplicationListener<PostUpdateEvent>, DataEventListener<PostUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<PostUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(PostUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<PostUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.QueuedRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueuedRemoveEventListener
|
||||
implements ApplicationListener<QueuedRemoveEvent>, DataEventListener<QueuedRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<QueuedRemoveEvent> sink =
|
||||
Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(QueuedRemoveEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<QueuedRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.QueuedUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueuedUpdateEventListener
|
||||
implements ApplicationListener<QueuedUpdateEvent>, DataEventListener<QueuedUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<QueuedUpdateEvent> sink =
|
||||
Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(QueuedUpdateEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<QueuedUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,10 @@ import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.Disposable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import tn.mnlr.vripper.download.DownloadJob;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.net.URI;
|
||||
@@ -40,22 +42,25 @@ public class ConnectionService {
|
||||
private int connectionTimeout;
|
||||
private int maxAttempts;
|
||||
|
||||
public ConnectionService(SettingsService settingsService) {
|
||||
public ConnectionService(EventBus eventBus, SettingsService settingsService) {
|
||||
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
|
||||
maxAttempts = settingsService.getSettings().getMaxAttempts();
|
||||
Flux<SettingsService.Settings> settingsFlux = settingsService.getSettingsFlux();
|
||||
disposable =
|
||||
settingsFlux.subscribe(
|
||||
settings -> {
|
||||
if (connectionTimeout != settings.getConnectionTimeout()) {
|
||||
connectionTimeout = settings.getConnectionTimeout();
|
||||
buildRequestConfig();
|
||||
}
|
||||
if (maxAttempts != settings.getMaxAttempts()) {
|
||||
maxAttempts = settings.getMaxAttempts();
|
||||
buildRetryPolicy();
|
||||
}
|
||||
});
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.map(e -> ((Settings) e.getData()))
|
||||
.subscribe(
|
||||
settings -> {
|
||||
if (connectionTimeout != settings.getConnectionTimeout()) {
|
||||
connectionTimeout = settings.getConnectionTimeout();
|
||||
buildRequestConfig();
|
||||
}
|
||||
if (maxAttempts != settings.getMaxAttempts()) {
|
||||
maxAttempts = settings.getMaxAttempts();
|
||||
buildRetryPolicy();
|
||||
}
|
||||
});
|
||||
|
||||
buildRequestConfig();
|
||||
buildRetryPolicy();
|
||||
|
||||
@@ -11,11 +11,11 @@ import tn.mnlr.vripper.jpa.repositories.IImageRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.impl.EventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.impl.LogEventRepository;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Transactional
|
||||
@@ -27,7 +27,7 @@ public class DataService {
|
||||
private final IQueuedRepository queuedRepository;
|
||||
private final IMetadataRepository metadataRepository;
|
||||
private final SettingsService settingsService;
|
||||
private final EventRepository eventRepository;
|
||||
private final LogEventRepository eventRepository;
|
||||
|
||||
@Autowired
|
||||
public DataService(
|
||||
@@ -36,7 +36,7 @@ public class DataService {
|
||||
IQueuedRepository queuedRepository,
|
||||
IMetadataRepository metadataRepository,
|
||||
SettingsService settingsService,
|
||||
EventRepository eventRepository) {
|
||||
LogEventRepository eventRepository) {
|
||||
this.postRepository = postRepository;
|
||||
this.imageRepository = imageRepository;
|
||||
this.queuedRepository = queuedRepository;
|
||||
@@ -45,6 +45,11 @@ public class DataService {
|
||||
this.eventRepository = eventRepository;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
sortPostsByRank();
|
||||
}
|
||||
|
||||
private void save(Post post) {
|
||||
postRepository.save(post);
|
||||
}
|
||||
@@ -68,10 +73,7 @@ public class DataService {
|
||||
image.setPostIdRef(post.getId());
|
||||
save(image);
|
||||
});
|
||||
}
|
||||
|
||||
public void setDownloadingToStopped() {
|
||||
postRepository.setDownloadingToStopped();
|
||||
sortPostsByRank();
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(Image image, Post post) {
|
||||
@@ -88,6 +90,26 @@ public class DataService {
|
||||
postRepository.updateDone(done, id);
|
||||
}
|
||||
|
||||
public void updatePostStatus(Status status, Long id) {
|
||||
postRepository.updateStatus(status, id);
|
||||
}
|
||||
|
||||
public void updatePostThanks(boolean thanked, Long id) {
|
||||
postRepository.updateThanked(thanked, id);
|
||||
}
|
||||
|
||||
public void updatePostTitle(String title, Long id) {
|
||||
postRepository.updateTitle(title, id);
|
||||
}
|
||||
|
||||
public void updatePostDownloadDirectory(String postFolderName, Long id) {
|
||||
postRepository.updateFolderName(postFolderName, id);
|
||||
}
|
||||
|
||||
public void updatePostRank(int rank, Long id) {
|
||||
postRepository.updateRank(rank, id);
|
||||
}
|
||||
|
||||
public void finishPost(@NonNull Post post) {
|
||||
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
|
||||
post.setStatus(Status.ERROR);
|
||||
@@ -100,16 +122,19 @@ public class DataService {
|
||||
post.setStatus(Status.COMPLETE);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
if (settingsService.getSettings().getClearCompleted()) {
|
||||
remove(post.getPostId());
|
||||
remove(List.of(post.getPostId()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void remove(@NonNull final String postId) {
|
||||
imageRepository.deleteAllByPostId(postId);
|
||||
metadataRepository.deleteByPostId(postId);
|
||||
postRepository.deleteByPostId(postId);
|
||||
private void remove(@NonNull final List<String> postIds) {
|
||||
for (String postId : postIds) {
|
||||
imageRepository.deleteAllByPostId(postId);
|
||||
metadataRepository.deleteByPostId(postId);
|
||||
postRepository.deleteByPostId(postId);
|
||||
}
|
||||
sortPostsByRank();
|
||||
}
|
||||
|
||||
public void newQueueLink(@NonNull final Queued queued) {
|
||||
@@ -122,18 +147,16 @@ public class DataService {
|
||||
|
||||
public List<String> clearCompleted() {
|
||||
List<String> completed = postRepository.findCompleted();
|
||||
completed.forEach(this::remove);
|
||||
remove(completed);
|
||||
return completed;
|
||||
}
|
||||
|
||||
public void removeAll(final List<String> postIds) {
|
||||
if (postIds != null && !postIds.isEmpty()) {
|
||||
for (String postId : postIds) {
|
||||
remove(postId);
|
||||
}
|
||||
} else {
|
||||
postRepository.findAll().forEach(p -> remove(p.getPostId()));
|
||||
}
|
||||
|
||||
remove(
|
||||
Objects.requireNonNullElse(
|
||||
postIds,
|
||||
postRepository.findAll().stream().map(Post::getPostId).collect(Collectors.toList())));
|
||||
}
|
||||
|
||||
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
|
||||
@@ -203,31 +226,24 @@ public class DataService {
|
||||
imageRepository.updateTotal(total, id);
|
||||
}
|
||||
|
||||
public void updatePostStatus(Status status, Long id) {
|
||||
postRepository.updateStatus(status, id);
|
||||
}
|
||||
|
||||
public void updateDownloadDirectory(String postFolderName, Long id) {
|
||||
postRepository.updateFolderName(postFolderName, id);
|
||||
}
|
||||
|
||||
public void updatePostTitle(String title, Long id) {
|
||||
postRepository.updateTitle(title, id);
|
||||
}
|
||||
|
||||
public void updatePostThanked(boolean thanked, Long id) {
|
||||
postRepository.updateThanked(thanked, id);
|
||||
}
|
||||
|
||||
public Optional<Event> findEventById(Long id) {
|
||||
public Optional<LogEvent> findEventById(Long id) {
|
||||
return eventRepository.findById(id);
|
||||
}
|
||||
|
||||
public List<Event> findAllEvents() {
|
||||
public List<LogEvent> findAllEvents() {
|
||||
return eventRepository.findAll();
|
||||
}
|
||||
|
||||
public void clearQueueLinks() {
|
||||
queuedRepository.deleteAll();
|
||||
}
|
||||
|
||||
private synchronized void sortPostsByRank() {
|
||||
List<Post> posts = findAllPosts();
|
||||
posts.sort(Comparator.comparing(Post::getAddedOn));
|
||||
for (int i = 0; i < posts.size(); i++) {
|
||||
posts.get(i).setRank(i);
|
||||
updatePostRank(i, posts.get(i).getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,11 +4,9 @@ import lombok.Getter;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Service
|
||||
@@ -16,12 +14,12 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
public class DownloadSpeedService {
|
||||
|
||||
private final AtomicLong read = new AtomicLong(0);
|
||||
private final Sinks.Many<Long> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
private final EventBus eventBus;
|
||||
@Getter private long currentValue;
|
||||
private boolean allowWrite = false;
|
||||
|
||||
public Flux<Long> getReadBytesPerSecond() {
|
||||
return sink.asFlux();
|
||||
public DownloadSpeedService(EventBus eventBus) {
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
public void increase(long read) {
|
||||
@@ -36,13 +34,8 @@ public class DownloadSpeedService {
|
||||
long newValue = read.getAndSet(0);
|
||||
if (newValue != currentValue) {
|
||||
currentValue = newValue;
|
||||
sink.emitNext(currentValue, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.BYTES_PER_SECOND, currentValue));
|
||||
}
|
||||
allowWrite = true;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,50 +5,38 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.download.PendingQueue;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.services.domain.GlobalState;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
public class GlobalStateService {
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final Sinks.Many<GlobalState> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
private final EventBus eventBus;
|
||||
@Getter private GlobalState currentState;
|
||||
|
||||
@Autowired
|
||||
public GlobalStateService(
|
||||
PendingQueue pendingQueue, DownloadService downloadService, DataService dataService) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
DownloadService downloadService, DataService dataService, EventBus eventBus) {
|
||||
this.downloadService = downloadService;
|
||||
this.dataService = dataService;
|
||||
}
|
||||
|
||||
public Flux<GlobalState> getGlobalState() {
|
||||
return sink.asFlux();
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 3000)
|
||||
private void interval() {
|
||||
GlobalState newGlobalState =
|
||||
new GlobalState(
|
||||
downloadService.runningCount(), pendingQueue.size(), dataService.countErrorImages());
|
||||
downloadService.runningCount(),
|
||||
downloadService.pendingCount(),
|
||||
dataService.countErrorImages());
|
||||
if (!newGlobalState.equals(currentState)) {
|
||||
currentState = newGlobalState;
|
||||
sink.emitNext(currentState, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,71 +1,24 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable;
|
||||
import tn.mnlr.vripper.tasks.MetadataRunnable;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MetadataService {
|
||||
|
||||
private static final List<String> dictionary =
|
||||
Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
|
||||
private final LoadingCache<Key, Metadata> cache;
|
||||
private final ConnectionService cm;
|
||||
private final VGAuthService VGAuthService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final XpathService xpathService;
|
||||
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
|
||||
private final ThreadPoolService threadPoolService;
|
||||
|
||||
public MetadataService(
|
||||
ConnectionService cm,
|
||||
VGAuthService VGAuthService,
|
||||
HtmlProcessorService htmlProcessorService,
|
||||
XpathService xpathService,
|
||||
ThreadPoolService threadPoolService) {
|
||||
this.cm = cm;
|
||||
this.VGAuthService = VGAuthService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
this.xpathService = xpathService;
|
||||
public MetadataService(ThreadPoolService threadPoolService) {
|
||||
this.threadPoolService = threadPoolService;
|
||||
|
||||
CacheLoader<Key, Metadata> loader =
|
||||
new CacheLoader<>() {
|
||||
@Override
|
||||
public Metadata load(@NonNull Key key) {
|
||||
return fetchMetadata(key);
|
||||
}
|
||||
};
|
||||
cache = CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.MINUTES).build(loader);
|
||||
}
|
||||
|
||||
public Metadata get(Post post) throws ExecutionException {
|
||||
Key key = new Key(post.getPostId(), post.getThreadId(), post.getUrl());
|
||||
return Metadata.from(cache.get(key));
|
||||
}
|
||||
|
||||
public void startFetchingMetadata(Post post) {
|
||||
@@ -74,138 +27,24 @@ public class MetadataService {
|
||||
fetchingMetadata.put(post.getPostId(), runnable);
|
||||
}
|
||||
|
||||
public void stopFetchingMetadata(Post post) {
|
||||
this.fetchingMetadata.forEach(
|
||||
(k, v) -> {
|
||||
if (k.equals(post.getPostId())) {
|
||||
v.setInterrupted(true);
|
||||
}
|
||||
});
|
||||
fetchingMetadata.remove(post.getPostId());
|
||||
}
|
||||
public void stopFetchingMetadata(List<String> postIds) {
|
||||
List<MetadataRunnable> stopping = new ArrayList<>();
|
||||
|
||||
private Metadata fetchMetadata(Key key) {
|
||||
HttpGet httpGet = cm.buildHttpGet(key.getUrl(), null);
|
||||
Metadata metadata = new Metadata();
|
||||
Failsafe.with(cm.getRetryPolicy())
|
||||
.onFailure(
|
||||
e -> {
|
||||
if (e.getFailure() instanceof InterruptedException
|
||||
|| e.getFailure().getCause() instanceof InterruptedException) {
|
||||
log.debug("Fetching interrupted");
|
||||
return;
|
||||
}
|
||||
log.error(
|
||||
String.format(
|
||||
"Error occurred when getting post metadata, postId %s", key.getPostId()),
|
||||
e.getFailure());
|
||||
})
|
||||
.run(
|
||||
() -> {
|
||||
HttpClient connection = cm.getClient().build();
|
||||
try (CloseableHttpResponse response =
|
||||
(CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(
|
||||
String.format(
|
||||
"Unexpected response code '%d' for %s",
|
||||
response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
try {
|
||||
if (Thread.interrupted()) {
|
||||
return;
|
||||
}
|
||||
Document document =
|
||||
htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
Node postNode =
|
||||
xpathService.getAsNode(
|
||||
document,
|
||||
String.format(
|
||||
"//li[@id='post_%s']/div[contains(@class, 'postdetails')]",
|
||||
key.getPostId()));
|
||||
String postedBy =
|
||||
xpathService
|
||||
.getAsNode(
|
||||
postNode,
|
||||
"./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font")
|
||||
.getTextContent()
|
||||
.trim();
|
||||
metadata.setPostedBy(postedBy);
|
||||
|
||||
Node node =
|
||||
xpathService.getAsNode(
|
||||
document, String.format("//div[@id='post_message_%s']", key.getPostId()));
|
||||
metadata.setResolvedNames(findTitleInContent(node));
|
||||
|
||||
metadata.setPostId(key.getPostId());
|
||||
} catch (Exception e) {
|
||||
throw new PostParseException(
|
||||
String.format(
|
||||
"Failed to parse thread %s, post %s", key.getThreadId(), key.getPostId()),
|
||||
e);
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
}
|
||||
}
|
||||
});
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private List<String> findTitleInContent(Node node) {
|
||||
List<String> altTitle = new ArrayList<>();
|
||||
findTitle(node, altTitle, new AtomicBoolean(true));
|
||||
return altTitle.stream().distinct().collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private void findTitle(Node node, List<String> altTitle, AtomicBoolean keepGoing) {
|
||||
if (!keepGoing.get()) {
|
||||
return;
|
||||
}
|
||||
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
|
||||
keepGoing.set(false);
|
||||
return;
|
||||
}
|
||||
if (node.getNodeType() == Node.ELEMENT_NODE) {
|
||||
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
|
||||
Node item = node.getChildNodes().item(i);
|
||||
findTitle(item, altTitle, keepGoing);
|
||||
if (!keepGoing.get()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
} else if (node.getNodeType() == Node.TEXT_NODE) {
|
||||
String text = node.getTextContent().trim();
|
||||
if (!text.isBlank()
|
||||
&& dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
|
||||
altTitle.add(text);
|
||||
for (Map.Entry<String, MetadataRunnable> entry : this.fetchingMetadata.entrySet()) {
|
||||
if (postIds.contains(entry.getKey())) {
|
||||
stopping.add(entry.getValue());
|
||||
entry.getValue().stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
static class Key {
|
||||
private final String postId;
|
||||
private final String threadId;
|
||||
private final String url;
|
||||
|
||||
Key(String postId, String threadId, String url) {
|
||||
this.postId = postId;
|
||||
this.threadId = threadId;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Key key = (Key) o;
|
||||
return Objects.equals(postId, key.postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(postId);
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(MetadataRunnable::isFinished);
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
postIds.forEach(fetchingMetadata::remove);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.exception.RenameException;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@@ -17,47 +18,43 @@ import java.util.concurrent.locks.ReentrantLock;
|
||||
@Slf4j
|
||||
public class PathService {
|
||||
|
||||
private final SettingsService settingsService;
|
||||
private final DataService dataService;
|
||||
|
||||
@Getter private final ReentrantLock directoryAccess = new ReentrantLock();
|
||||
|
||||
public PathService(SettingsService settingsService, DataService dataService) {
|
||||
this.settingsService = settingsService;
|
||||
public PathService(DataService dataService) {
|
||||
this.dataService = dataService;
|
||||
}
|
||||
|
||||
public final File calcDownloadDirectory(Post post) {
|
||||
return new File(settingsService.getSettings().getDownloadPath(), post.getDownloadDirectory());
|
||||
public final File calcDownloadDirectory(Post post, Settings settings) {
|
||||
return new File(settings.getDownloadPath(), post.getDownloadDirectory());
|
||||
}
|
||||
|
||||
private File getRootFolder(@NonNull String forum, @NonNull String threadTitle) {
|
||||
private File getRootFolder(
|
||||
@NonNull String forum, @NonNull String threadTitle, Settings settings) {
|
||||
File sourceFolder =
|
||||
settingsService.getSettings().getSubLocation()
|
||||
? new File(settingsService.getSettings().getDownloadPath(), sanitize(forum))
|
||||
: new File(settingsService.getSettings().getDownloadPath());
|
||||
return settingsService.getSettings().getThreadSubLocation()
|
||||
? new File(sourceFolder, threadTitle)
|
||||
: sourceFolder;
|
||||
settings.getSubLocation()
|
||||
? new File(settings.getDownloadPath(), sanitize(forum))
|
||||
: new File(settings.getDownloadPath());
|
||||
return settings.getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
|
||||
}
|
||||
|
||||
public final void createDefaultPostFolder(Post post) {
|
||||
File downloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle());
|
||||
public final void createDefaultPostFolder(Post post, Settings settings) {
|
||||
File downloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
|
||||
downloadDirectory =
|
||||
new File(
|
||||
downloadDirectory,
|
||||
settingsService.getSettings().getAppendPostId()
|
||||
settings.getAppendPostId()
|
||||
? sanitize(post.getTitle()) + "_" + post.getPostId()
|
||||
: sanitize(post.getTitle()));
|
||||
downloadDirectory = makeDir(downloadDirectory);
|
||||
post.setDownloadDirectory(
|
||||
downloadDirectory
|
||||
.getAbsolutePath()
|
||||
.replace(settingsService.getSettings().getDownloadPath(), ""));
|
||||
dataService.updateDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
downloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
|
||||
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
}
|
||||
|
||||
public final void rename(@NonNull String postId, @NonNull String altName) throws RenameException {
|
||||
public final void rename(@NonNull String postId, @NonNull String altName, Settings settings)
|
||||
throws RenameException {
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (altName.equals(post.getTitle())) {
|
||||
return;
|
||||
@@ -70,9 +67,9 @@ public class PathService {
|
||||
return;
|
||||
}
|
||||
|
||||
File newDownloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle());
|
||||
File newDownloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
|
||||
newDownloadDirectory = new File(newDownloadDirectory, sanitize(altName));
|
||||
File currentDownloadDirectory = calcDownloadDirectory(post);
|
||||
File currentDownloadDirectory = calcDownloadDirectory(post, settings);
|
||||
try {
|
||||
directoryAccess.lock();
|
||||
Files.move(
|
||||
@@ -80,10 +77,8 @@ public class PathService {
|
||||
newDownloadDirectory.toPath(),
|
||||
StandardCopyOption.ATOMIC_MOVE);
|
||||
post.setDownloadDirectory(
|
||||
newDownloadDirectory
|
||||
.getAbsolutePath()
|
||||
.replace(settingsService.getSettings().getDownloadPath(), ""));
|
||||
dataService.updateDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
newDownloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
|
||||
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
|
||||
post.setTitle(altName);
|
||||
dataService.updatePostTitle(post.getTitle(), post.getId());
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import reactor.core.Disposable;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanParser;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddQueuedRunnable;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -24,31 +25,30 @@ public class PostService {
|
||||
|
||||
private final DataService dataService;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final MetadataService metadataService;
|
||||
private final LoadingCache<Queued, MultiPostScanResult> cache;
|
||||
private final Disposable disposable;
|
||||
|
||||
@Autowired
|
||||
public PostService(
|
||||
DataService dataService,
|
||||
ThreadPoolService threadPoolService,
|
||||
MetadataService metadataService) {
|
||||
DataService dataService, ThreadPoolService threadPoolService, EventBus eventBus) {
|
||||
this.dataService = dataService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.metadataService = metadataService;
|
||||
|
||||
CacheLoader<Queued, MultiPostScanResult> loader =
|
||||
new CacheLoader<>() {
|
||||
@Override
|
||||
public MultiPostScanResult load(@NonNull Queued multiPostItem) throws Exception {
|
||||
return new MultiPostScanParser(multiPostItem).parse();
|
||||
}
|
||||
};
|
||||
|
||||
cache = CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.MINUTES).build(loader);
|
||||
cache =
|
||||
Caffeine.newBuilder()
|
||||
.expireAfterWrite(5, TimeUnit.MINUTES)
|
||||
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.subscribe(e -> this.cache.invalidateAll());
|
||||
}
|
||||
|
||||
public void stopFetchingMetadata(Post post) {
|
||||
metadataService.stopFetchingMetadata(post);
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
if (disposable != null) {
|
||||
disposable.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public void processMultiPost(List<Queued> queuedList) {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
@@ -13,11 +12,11 @@ import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.exception.ValidationException;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
@@ -46,7 +45,7 @@ public class SettingsService {
|
||||
|
||||
private final Set<String> proxies = new HashSet<>();
|
||||
|
||||
private Sinks.Many<Settings> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Getter private Settings settings = new Settings();
|
||||
|
||||
@@ -54,7 +53,10 @@ public class SettingsService {
|
||||
private Resource defaultProxies;
|
||||
|
||||
public SettingsService(
|
||||
@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
|
||||
@Value("${base.dir}") String baseDir,
|
||||
@Value("${base.dir.name}") String baseDirName,
|
||||
EventBus eventBus) {
|
||||
this.eventBus = eventBus;
|
||||
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
|
||||
this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
|
||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
@@ -64,7 +66,7 @@ public class SettingsService {
|
||||
private void init() {
|
||||
loadViperProxies();
|
||||
restore();
|
||||
sink.emitNext(settings, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
|
||||
}
|
||||
|
||||
private void loadViperProxies() {
|
||||
@@ -93,10 +95,6 @@ public class SettingsService {
|
||||
proxies.add("https://vipergirls.to");
|
||||
}
|
||||
|
||||
Flux<Settings> getSettingsFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
public List<String> getProxies() {
|
||||
return new ArrayList<>(proxies);
|
||||
}
|
||||
@@ -116,7 +114,7 @@ public class SettingsService {
|
||||
this.settings = settings;
|
||||
|
||||
save();
|
||||
sink.emitNext(settings, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
|
||||
}
|
||||
|
||||
public void restore() {
|
||||
@@ -232,7 +230,6 @@ public class SettingsService {
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
save();
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
public void check(Settings settings) throws ValidationException {
|
||||
@@ -305,71 +302,4 @@ public class SettingsService {
|
||||
this.darkTheme = darkTheme;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public static class Settings {
|
||||
|
||||
@JsonProperty("downloadPath")
|
||||
private String downloadPath;
|
||||
|
||||
@JsonProperty("maxThreads")
|
||||
private Integer maxThreads;
|
||||
|
||||
@JsonProperty("maxTotalThreads")
|
||||
private Integer maxTotalThreads;
|
||||
|
||||
@JsonProperty("autoStart")
|
||||
private Boolean autoStart;
|
||||
|
||||
@JsonProperty("vLogin")
|
||||
private Boolean vLogin;
|
||||
|
||||
@JsonProperty("vUsername")
|
||||
private String vUsername;
|
||||
|
||||
@JsonProperty("vPassword")
|
||||
private String vPassword;
|
||||
|
||||
@JsonProperty("vThanks")
|
||||
private Boolean vThanks;
|
||||
|
||||
@JsonProperty("desktopClipboard")
|
||||
private Boolean desktopClipboard;
|
||||
|
||||
@JsonProperty("forceOrder")
|
||||
private Boolean forceOrder;
|
||||
|
||||
@JsonProperty("subLocation")
|
||||
private Boolean subLocation;
|
||||
|
||||
@JsonProperty("threadSubLocation")
|
||||
private Boolean threadSubLocation;
|
||||
|
||||
@JsonProperty("clearCompleted")
|
||||
private Boolean clearCompleted;
|
||||
|
||||
@JsonProperty("darkTheme")
|
||||
private Boolean darkTheme;
|
||||
|
||||
@JsonProperty("appendPostId")
|
||||
private Boolean appendPostId;
|
||||
|
||||
@JsonProperty("leaveThanksOnStart")
|
||||
private Boolean leaveThanksOnStart;
|
||||
|
||||
@JsonProperty("connectionTimeout")
|
||||
private Integer connectionTimeout;
|
||||
|
||||
@JsonProperty("maxAttempts")
|
||||
private Integer maxAttempts;
|
||||
|
||||
@JsonProperty("vProxy")
|
||||
private String vProxy;
|
||||
|
||||
@JsonProperty("maxEventLog")
|
||||
private Integer maxEventLog;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,12 +15,11 @@ import org.apache.http.util.EntityUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.Disposable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.exception.VripperException;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable;
|
||||
import tn.mnlr.vripper.tasks.LeaveThanksRunnable;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
@@ -38,17 +37,26 @@ public class VGAuthService {
|
||||
private final Disposable disposable;
|
||||
|
||||
@Getter private final HttpClientContext context = HttpClientContext.create();
|
||||
private final Sinks.Many<String> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
@Getter private boolean authenticated = false;
|
||||
@Getter private String loggedUser = "";
|
||||
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public VGAuthService(
|
||||
ConnectionService cm, SettingsService settingsService, ThreadPoolService threadPoolService) {
|
||||
ConnectionService cm,
|
||||
SettingsService settingsService,
|
||||
ThreadPoolService threadPoolService,
|
||||
EventBus eventBus) {
|
||||
this.cm = cm;
|
||||
this.settingsService = settingsService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate());
|
||||
this.eventBus = eventBus;
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.subscribe(e -> authenticate());
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
@@ -59,14 +67,9 @@ public class VGAuthService {
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
disposable.dispose();
|
||||
}
|
||||
|
||||
public Flux<String> getLoggedInUser() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
public void authenticate() {
|
||||
|
||||
log.info("Authenticating using ViperGirls credentials");
|
||||
@@ -76,7 +79,7 @@ public class VGAuthService {
|
||||
log.debug("Authentication option is disabled");
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
sink.emitNext(loggedUser, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -87,7 +90,7 @@ public class VGAuthService {
|
||||
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
sink.emitNext(loggedUser, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -104,7 +107,7 @@ public class VGAuthService {
|
||||
} catch (Exception e) {
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
sink.emitNext(loggedUser, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
|
||||
log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
|
||||
return;
|
||||
}
|
||||
@@ -138,14 +141,14 @@ public class VGAuthService {
|
||||
} catch (Exception e) {
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
sink.emitNext(loggedUser, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
|
||||
log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
|
||||
return;
|
||||
}
|
||||
authenticated = true;
|
||||
loggedUser = username;
|
||||
log.info(String.format("Authenticated: %s", username));
|
||||
sink.emitNext(loggedUser, EmitHandler.RETRY);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
|
||||
}
|
||||
|
||||
public void leaveThanks(Post post) {
|
||||
|
||||
@@ -2,6 +2,8 @@ package tn.mnlr.vripper.services.domain;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
@Getter
|
||||
public class DownloadSpeed {
|
||||
|
||||
@@ -22,4 +24,17 @@ public class DownloadSpeed {
|
||||
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
|
||||
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
DownloadSpeed that = (DownloadSpeed) o;
|
||||
return Objects.equals(speed, that.speed);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(speed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,8 +68,6 @@ public class MultiPostScanParser {
|
||||
}
|
||||
|
||||
try {
|
||||
//
|
||||
// System.out.println(EntityUtils.toString(response.getEntity()));
|
||||
factory
|
||||
.newSAXParser()
|
||||
.parse(
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package tn.mnlr.vripper.services.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Slf4j
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class Settings implements Cloneable {
|
||||
|
||||
@JsonProperty("downloadPath")
|
||||
private String downloadPath;
|
||||
|
||||
@JsonProperty("maxThreads")
|
||||
private Integer maxThreads;
|
||||
|
||||
@JsonProperty("maxTotalThreads")
|
||||
private Integer maxTotalThreads;
|
||||
|
||||
@JsonProperty("autoStart")
|
||||
private Boolean autoStart;
|
||||
|
||||
@JsonProperty("vLogin")
|
||||
private Boolean vLogin;
|
||||
|
||||
@JsonProperty("vUsername")
|
||||
private String vUsername;
|
||||
|
||||
@JsonProperty("vPassword")
|
||||
private String vPassword;
|
||||
|
||||
@JsonProperty("vThanks")
|
||||
private Boolean vThanks;
|
||||
|
||||
@JsonProperty("desktopClipboard")
|
||||
private Boolean desktopClipboard;
|
||||
|
||||
@JsonProperty("forceOrder")
|
||||
private Boolean forceOrder;
|
||||
|
||||
@JsonProperty("subLocation")
|
||||
private Boolean subLocation;
|
||||
|
||||
@JsonProperty("threadSubLocation")
|
||||
private Boolean threadSubLocation;
|
||||
|
||||
@JsonProperty("clearCompleted")
|
||||
private Boolean clearCompleted;
|
||||
|
||||
@JsonProperty("darkTheme")
|
||||
private Boolean darkTheme;
|
||||
|
||||
@JsonProperty("appendPostId")
|
||||
private Boolean appendPostId;
|
||||
|
||||
@JsonProperty("leaveThanksOnStart")
|
||||
private Boolean leaveThanksOnStart;
|
||||
|
||||
@JsonProperty("connectionTimeout")
|
||||
private Integer connectionTimeout;
|
||||
|
||||
@JsonProperty("maxAttempts")
|
||||
private Integer maxAttempts;
|
||||
|
||||
@JsonProperty("vProxy")
|
||||
private String vProxy;
|
||||
|
||||
@JsonProperty("maxEventLog")
|
||||
private Integer maxEventLog;
|
||||
|
||||
@Override
|
||||
public Object clone() {
|
||||
Settings clone;
|
||||
try {
|
||||
clone = (Settings) super.clone();
|
||||
} catch (CloneNotSupportedException e) {
|
||||
log.error(e.getMessage(), e);
|
||||
clone = new Settings();
|
||||
}
|
||||
clone.downloadPath = downloadPath;
|
||||
clone.maxThreads = maxThreads;
|
||||
clone.maxTotalThreads = maxTotalThreads;
|
||||
clone.autoStart = autoStart;
|
||||
clone.vLogin = vLogin;
|
||||
clone.vUsername = vUsername;
|
||||
clone.vPassword = vPassword;
|
||||
clone.vThanks = vThanks;
|
||||
clone.desktopClipboard = desktopClipboard;
|
||||
clone.forceOrder = forceOrder;
|
||||
clone.subLocation = subLocation;
|
||||
clone.threadSubLocation = threadSubLocation;
|
||||
clone.clearCompleted = clearCompleted;
|
||||
clone.darkTheme = darkTheme;
|
||||
clone.appendPostId = appendPostId;
|
||||
clone.leaveThanksOnStart = leaveThanksOnStart;
|
||||
clone.connectionTimeout = connectionTimeout;
|
||||
clone.maxAttempts = maxAttempts;
|
||||
clone.vProxy = vProxy;
|
||||
clone.maxEventLog = maxEventLog;
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.MetadataService;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
|
||||
|
||||
@Slf4j
|
||||
public class MetadataRunnable implements Runnable {
|
||||
|
||||
@Getter private final Post post;
|
||||
|
||||
private final MetadataService metadataService;
|
||||
private final DataService dataService;
|
||||
private final IEventRepository eventRepository;
|
||||
|
||||
private final Event event;
|
||||
|
||||
@Setter private volatile boolean interrupted = false;
|
||||
|
||||
public MetadataRunnable(@NonNull Post post) {
|
||||
this.post = post;
|
||||
metadataService = SpringContext.getBean(MetadataService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
eventRepository = SpringContext.getBean(IEventRepository.class);
|
||||
event =
|
||||
new Event(
|
||||
Event.Type.METADATA,
|
||||
Event.Status.PENDING,
|
||||
LocalDateTime.now(),
|
||||
"Fetching metadata for " + post.getUrl());
|
||||
eventRepository.save(event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
try {
|
||||
event.setStatus(Event.Status.PROCESSING);
|
||||
eventRepository.update(event);
|
||||
if (interrupted) {
|
||||
String message = String.format("Fetching metadata for %s interrupted", post.getUrl());
|
||||
event.setStatus(Event.Status.DONE);
|
||||
event.setMessage(message);
|
||||
eventRepository.update(event);
|
||||
return;
|
||||
}
|
||||
|
||||
Metadata metadata = metadataService.get(post);
|
||||
if (metadata == null) {
|
||||
String message = String.format("Fetching metadata for %s failed", post.getUrl());
|
||||
event.setStatus(ERROR);
|
||||
event.setMessage(message);
|
||||
eventRepository.update(event);
|
||||
return;
|
||||
}
|
||||
dataService.setMetadata(post, metadata);
|
||||
|
||||
event.setStatus(Event.Status.DONE);
|
||||
eventRepository.update(event);
|
||||
} catch (Exception e) {
|
||||
String message = String.format("Failed to fetch metadata for %s", post.getUrl());
|
||||
log.error(message, e);
|
||||
event.setMessage(message + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
+39
-44
@@ -1,15 +1,15 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.download.PendingQueue;
|
||||
import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.MetadataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
@@ -18,10 +18,11 @@ import tn.mnlr.vripper.services.domain.PostScanParser;
|
||||
import tn.mnlr.vripper.services.domain.PostScanResult;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
|
||||
import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING;
|
||||
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
|
||||
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.PROCESSING;
|
||||
|
||||
@Slf4j
|
||||
public class AddPostRunnable implements Runnable {
|
||||
@@ -31,11 +32,11 @@ public class AddPostRunnable implements Runnable {
|
||||
private final DataService dataService;
|
||||
private final MetadataService metadataService;
|
||||
private final SettingsService settingsService;
|
||||
private final PendingQueue pendingQueue;
|
||||
private final VGAuthService VGAuthService;
|
||||
private final Event event;
|
||||
private final IEventRepository eventRepository;
|
||||
private final LogEvent logEvent;
|
||||
private final ILogEventRepository eventRepository;
|
||||
private final String link;
|
||||
private final DownloadService downloadService;
|
||||
|
||||
public AddPostRunnable(String postId, String threadId) {
|
||||
this.postId = postId;
|
||||
@@ -43,32 +44,32 @@ public class AddPostRunnable implements Runnable {
|
||||
this.dataService = SpringContext.getBean(DataService.class);
|
||||
this.metadataService = SpringContext.getBean(MetadataService.class);
|
||||
this.settingsService = SpringContext.getBean(SettingsService.class);
|
||||
this.pendingQueue = SpringContext.getBean(PendingQueue.class);
|
||||
this.downloadService = SpringContext.getBean(DownloadService.class);
|
||||
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
|
||||
this.eventRepository = SpringContext.getBean(IEventRepository.class);
|
||||
this.eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
link =
|
||||
settingsService.getSettings().getVProxy()
|
||||
+ String.format("/threads/%s?%s", threadId, (postId != null ? "p=" + postId : ""));
|
||||
event =
|
||||
new Event(
|
||||
Event.Type.POST,
|
||||
Event.Status.PENDING,
|
||||
logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.POST,
|
||||
LogEvent.Status.PENDING,
|
||||
LocalDateTime.now(),
|
||||
String.format("Processing %s", link));
|
||||
eventRepository.save(event);
|
||||
eventRepository.save(logEvent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
try {
|
||||
event.setStatus(PROCESSING);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(PROCESSING);
|
||||
eventRepository.update(logEvent);
|
||||
if (dataService.exists(postId)) {
|
||||
log.warn(String.format("skipping %s, already loaded", postId));
|
||||
event.setMessage(String.format("Gallery %s is already loaded", link));
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(String.format("Gallery %s is already loaded", link));
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -80,25 +81,25 @@ public class AddPostRunnable implements Runnable {
|
||||
} catch (PostParseException e) {
|
||||
String error = String.format("parsing failed for gallery %s", link);
|
||||
log.error(error, e);
|
||||
event.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
if (postScanResult.getPost().isEmpty()) {
|
||||
String error = String.format("Gallery %s contains no galleries", link);
|
||||
log.error(error);
|
||||
event.setMessage(error);
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error);
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
if (postScanResult.getImages().isEmpty()) {
|
||||
String error = String.format("Gallery %s contains no images to download", link);
|
||||
log.error(error);
|
||||
event.setMessage(error);
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error);
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -106,19 +107,12 @@ public class AddPostRunnable implements Runnable {
|
||||
Set<Image> images = postScanResult.getImages();
|
||||
|
||||
dataService.newPost(post, images);
|
||||
|
||||
metadataService.startFetchingMetadata(post);
|
||||
|
||||
if (settingsService.getSettings().getAutoStart()) {
|
||||
log.debug("Auto start downloads option is enabled");
|
||||
post.setStatus(Status.PENDING);
|
||||
try {
|
||||
pendingQueue.enqueue(post, images);
|
||||
} catch (InterruptedException e) {
|
||||
log.warn("Interruption was caught");
|
||||
Thread.currentThread().interrupt();
|
||||
return;
|
||||
}
|
||||
downloadService.enqueue(Map.of(post, images));
|
||||
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
|
||||
} else {
|
||||
post.setStatus(Status.STOPPED);
|
||||
@@ -129,15 +123,16 @@ public class AddPostRunnable implements Runnable {
|
||||
VGAuthService.leaveThanks(post);
|
||||
}
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
event.setMessage(String.format("Gallery %s is successfully added to download queue", link));
|
||||
event.setStatus(Event.Status.DONE);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(
|
||||
String.format("Gallery %s is successfully added to download queue", link));
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
eventRepository.update(logEvent);
|
||||
} catch (Exception e) {
|
||||
String error = String.format("Error when adding gallery %s", link);
|
||||
log.error(error, e);
|
||||
event.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
-34
@@ -1,11 +1,11 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.ThreadPoolService;
|
||||
@@ -13,8 +13,8 @@ import tn.mnlr.vripper.services.domain.MultiPostScanResult;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
|
||||
import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING;
|
||||
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
|
||||
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.PROCESSING;
|
||||
|
||||
@Slf4j
|
||||
public class AddQueuedRunnable implements Runnable {
|
||||
@@ -22,49 +22,49 @@ public class AddQueuedRunnable implements Runnable {
|
||||
private final Queued queued;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final DataService dataService;
|
||||
private final IEventRepository eventRepository;
|
||||
private final ILogEventRepository eventRepository;
|
||||
private final PostService postService;
|
||||
private final Event event;
|
||||
private final LogEvent logEvent;
|
||||
|
||||
public AddQueuedRunnable(Queued queued) {
|
||||
this.queued = queued;
|
||||
threadPoolService = SpringContext.getBean(ThreadPoolService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
eventRepository = SpringContext.getBean(IEventRepository.class);
|
||||
eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
postService = SpringContext.getBean(PostService.class);
|
||||
event =
|
||||
new Event(
|
||||
Event.Type.QUEUED,
|
||||
Event.Status.PENDING,
|
||||
logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.QUEUED,
|
||||
LogEvent.Status.PENDING,
|
||||
LocalDateTime.now(),
|
||||
String.format("Processing multi-post link %s", queued.getLink()));
|
||||
eventRepository.save(event);
|
||||
eventRepository.save(logEvent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
event.setStatus(PROCESSING);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(PROCESSING);
|
||||
eventRepository.update(logEvent);
|
||||
MultiPostScanResult multiPostScanResult = postService.get(queued);
|
||||
if (multiPostScanResult == null) {
|
||||
String message = String.format("Fetching multi-post link %s failed", queued.getLink());
|
||||
event.setStatus(ERROR);
|
||||
event.setMessage(message);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(ERROR);
|
||||
logEvent.setMessage(message);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
} else if (multiPostScanResult.getError() != null) {
|
||||
String message =
|
||||
"Nothing found for " + queued.getLink() + "\n" + multiPostScanResult.getError();
|
||||
event.setStatus(ERROR);
|
||||
event.setMessage(message);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(ERROR);
|
||||
logEvent.setMessage(message);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
} else if (multiPostScanResult.getPosts().isEmpty()) {
|
||||
String message = "Nothing found for " + queued.getLink();
|
||||
event.setStatus(ERROR);
|
||||
event.setMessage(message);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(ERROR);
|
||||
logEvent.setMessage(message);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
queued.setTotal(multiPostScanResult.getPosts().size());
|
||||
@@ -76,27 +76,28 @@ public class AddQueuedRunnable implements Runnable {
|
||||
new AddPostRunnable(
|
||||
multiPostScanResult.getPosts().get(0).getPostId(),
|
||||
multiPostScanResult.getPosts().get(0).getThreadId()));
|
||||
event.setStatus(Event.Status.DONE);
|
||||
event.setMessage(String.format("Link %s is added to download queue", queued.getLink()));
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
logEvent.setMessage(String.format("Link %s is added to download queue", queued.getLink()));
|
||||
} else {
|
||||
if (dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) {
|
||||
dataService.newQueueLink(queued);
|
||||
event.setStatus(Event.Status.DONE);
|
||||
event.setMessage(String.format("Link %s is added to multi-post links", queued.getLink()));
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
logEvent.setMessage(
|
||||
String.format("Link %s is added to multi-post links", queued.getLink()));
|
||||
} else {
|
||||
log.info(String.format("Link %s is already loaded", queued.getLink()));
|
||||
event.setStatus(Event.Status.ERROR);
|
||||
event.setMessage(
|
||||
logEvent.setStatus(LogEvent.Status.ERROR);
|
||||
logEvent.setMessage(
|
||||
String.format("%s has already been added to multi-post links", queued.getLink()));
|
||||
}
|
||||
}
|
||||
eventRepository.update(event);
|
||||
eventRepository.update(logEvent);
|
||||
} catch (Exception e) {
|
||||
String error = String.format("Error when adding multi-post link %s", queued.getLink());
|
||||
log.error(error, e);
|
||||
event.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
+34
-34
@@ -1,4 +1,4 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.NameValuePair;
|
||||
@@ -11,9 +11,9 @@ import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
@@ -30,10 +30,10 @@ public class LeaveThanksRunnable implements Runnable {
|
||||
private final DataService dataService;
|
||||
private final HttpClientContext context;
|
||||
private final SettingsService settingsService;
|
||||
private final IEventRepository eventRepository;
|
||||
private final ILogEventRepository eventRepository;
|
||||
private final boolean authenticated;
|
||||
private final Post post;
|
||||
private final Event event;
|
||||
private final LogEvent logEvent;
|
||||
|
||||
public LeaveThanksRunnable(Post post, boolean authenticated, HttpClientContext context) {
|
||||
this.post = post;
|
||||
@@ -41,51 +41,51 @@ public class LeaveThanksRunnable implements Runnable {
|
||||
this.context = context;
|
||||
cm = SpringContext.getBean(ConnectionService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
eventRepository = SpringContext.getBean(IEventRepository.class);
|
||||
eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
settingsService = SpringContext.getBean(SettingsService.class);
|
||||
event =
|
||||
new Event(
|
||||
Event.Type.THANKS,
|
||||
Event.Status.PENDING,
|
||||
logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.THANKS,
|
||||
LogEvent.Status.PENDING,
|
||||
LocalDateTime.now(),
|
||||
String.format("Leaving thanks for %s", post.getUrl()));
|
||||
eventRepository.save(event);
|
||||
eventRepository.save(logEvent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
event.setStatus(Event.Status.PROCESSING);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.PROCESSING);
|
||||
eventRepository.update(logEvent);
|
||||
if (!settingsService.getSettings().getVLogin()) {
|
||||
event.setMessage(
|
||||
logEvent.setMessage(
|
||||
String.format(
|
||||
"Will not send a like for %s\nAuthentication with ViperGirls option is disabled",
|
||||
post.getUrl()));
|
||||
event.setStatus(Event.Status.DONE);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
if (!settingsService.getSettings().getVThanks()) {
|
||||
event.setMessage(
|
||||
logEvent.setMessage(
|
||||
String.format(
|
||||
"Will not send a like for %s\nLeave thanks option is disabled", post.getUrl()));
|
||||
event.setStatus(Event.Status.DONE);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
if (!authenticated) {
|
||||
event.setMessage(
|
||||
logEvent.setMessage(
|
||||
String.format("Will not send a like for %s\nYou are not authenticated", post.getUrl()));
|
||||
event.setStatus(Event.Status.ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
if (post.isThanked()) {
|
||||
event.setMessage(
|
||||
logEvent.setMessage(
|
||||
String.format("Will not send a like for %s\nAlready left a thanks", post.getUrl()));
|
||||
event.setStatus(Event.Status.DONE);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -101,9 +101,9 @@ public class LeaveThanksRunnable implements Runnable {
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
String error = String.format("Request error for %s", post.getUrl());
|
||||
log.error(error, e);
|
||||
event.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(Event.Status.ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(LogEvent.Status.ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -117,18 +117,18 @@ public class LeaveThanksRunnable implements Runnable {
|
||||
try (CloseableHttpResponse response = client.execute(postThanks, context)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 == 2) {
|
||||
post.setThanked(true);
|
||||
dataService.updatePostThanked(post.isThanked(), post.getId());
|
||||
dataService.updatePostThanks(post.isThanked(), post.getId());
|
||||
}
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
}
|
||||
event.setStatus(Event.Status.DONE);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
eventRepository.update(logEvent);
|
||||
} catch (Exception e) {
|
||||
String error = String.format("Failed to leave a thanks for %s", post.getUrl());
|
||||
log.error(error, e);
|
||||
event.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(Event.Status.ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(LogEvent.Status.ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
-21
@@ -1,12 +1,12 @@
|
||||
package tn.mnlr.vripper.services.domain.tasks;
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.lang.NonNull;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.jpa.repositories.impl.EventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.impl.LogEventRepository;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
@@ -16,7 +16,7 @@ import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
|
||||
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
|
||||
|
||||
@Slf4j
|
||||
public class LinkScanRunnable implements Runnable {
|
||||
@@ -25,29 +25,29 @@ public class LinkScanRunnable implements Runnable {
|
||||
private final List<String> urlList;
|
||||
private final SettingsService settingsService;
|
||||
private final PostService postService;
|
||||
private final EventRepository eventRepository;
|
||||
private final Event event;
|
||||
private final LogEventRepository eventRepository;
|
||||
private final LogEvent logEvent;
|
||||
|
||||
public LinkScanRunnable(@NonNull List<String> urlList) {
|
||||
this.urlList = urlList;
|
||||
settingsService = SpringContext.getBean(SettingsService.class);
|
||||
postService = SpringContext.getBean(PostService.class);
|
||||
eventRepository = SpringContext.getBean(EventRepository.class);
|
||||
event =
|
||||
new Event(
|
||||
Event.Type.SCAN,
|
||||
Event.Status.PENDING,
|
||||
eventRepository = SpringContext.getBean(LogEventRepository.class);
|
||||
logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.SCAN,
|
||||
LogEvent.Status.PENDING,
|
||||
LocalDateTime.now(),
|
||||
"Links to scan:\n\t" + String.join("\n\t", urlList));
|
||||
eventRepository.save(event);
|
||||
eventRepository.save(logEvent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
synchronized (LOCK) {
|
||||
try {
|
||||
event.setStatus(Event.Status.PROCESSING);
|
||||
eventRepository.update(event);
|
||||
logEvent.setStatus(LogEvent.Status.PROCESSING);
|
||||
eventRepository.update(logEvent);
|
||||
ArrayList<Queued> queuedList = new ArrayList<>();
|
||||
List<String> unsupported = new ArrayList<>();
|
||||
List<String> unrecognized = new ArrayList<>();
|
||||
@@ -91,18 +91,18 @@ public class LinkScanRunnable implements Runnable {
|
||||
|
||||
postService.processMultiPost(queuedList);
|
||||
if (!unsupported.isEmpty() || !unrecognized.isEmpty()) {
|
||||
event.setStatus(Event.Status.ERROR);
|
||||
event.setMessage("Some links failed to be scanned: \n" + errorMessage);
|
||||
logEvent.setStatus(LogEvent.Status.ERROR);
|
||||
logEvent.setMessage("Some links failed to be scanned: \n" + errorMessage);
|
||||
} else {
|
||||
event.setStatus(Event.Status.DONE);
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
}
|
||||
eventRepository.update(event);
|
||||
eventRepository.update(logEvent);
|
||||
} catch (Exception e) {
|
||||
String error = "Error when scanning links";
|
||||
log.error(error, e);
|
||||
event.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
event.setStatus(ERROR);
|
||||
eventRepository.update(event);
|
||||
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package tn.mnlr.vripper.tasks;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.download.DownloadJob;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static tn.mnlr.vripper.jpa.domain.LogEvent.Status.ERROR;
|
||||
|
||||
@Slf4j
|
||||
public class MetadataRunnable implements Runnable {
|
||||
|
||||
private static final List<String> dictionary =
|
||||
Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
|
||||
|
||||
@Getter private final Post post;
|
||||
|
||||
private final DataService dataService;
|
||||
private final ILogEventRepository eventRepository;
|
||||
|
||||
private final LogEvent logEvent;
|
||||
private final ConnectionService cm;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
private final HttpClientContext context = HttpClientContext.create();
|
||||
private final MetadataService metadataService;
|
||||
private volatile boolean stopped = false;
|
||||
@Getter private volatile boolean finished = false;
|
||||
|
||||
public MetadataRunnable(@NonNull Post post) {
|
||||
this.post = post;
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
cm = SpringContext.getBean(ConnectionService.class);
|
||||
VGAuthService vgAuthService = SpringContext.getBean(VGAuthService.class);
|
||||
htmlProcessorService = SpringContext.getBean(HtmlProcessorService.class);
|
||||
xpathService = SpringContext.getBean(XpathService.class);
|
||||
metadataService = SpringContext.getBean(MetadataService.class);
|
||||
|
||||
context.setCookieStore(vgAuthService.getContext().getCookieStore());
|
||||
context.setAttribute(
|
||||
DownloadJob.ContextAttributes.OPEN_CONNECTION.toString(),
|
||||
Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
|
||||
|
||||
logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.METADATA,
|
||||
LogEvent.Status.PENDING,
|
||||
LocalDateTime.now(),
|
||||
"Fetching metadata for " + post.getUrl());
|
||||
eventRepository.save(logEvent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
try {
|
||||
logEvent.setStatus(LogEvent.Status.PROCESSING);
|
||||
eventRepository.update(logEvent);
|
||||
Metadata metadata = fetchMetadata(post.getPostId(), post.getThreadId(), post.getUrl());
|
||||
if (metadata != null && !stopped) {
|
||||
dataService.setMetadata(post, metadata);
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
} else {
|
||||
logEvent.setStatus(ERROR);
|
||||
logEvent.setMessage(String.format("Fetching metadata for %s failed", post.getUrl()));
|
||||
}
|
||||
eventRepository.update(logEvent);
|
||||
} catch (Exception e) {
|
||||
String message = String.format("Failed to fetch metadata for %s", post.getUrl());
|
||||
log.error(message, e);
|
||||
logEvent.setMessage(message + "\n" + Utils.throwableToString(e));
|
||||
logEvent.setStatus(ERROR);
|
||||
eventRepository.update(logEvent);
|
||||
} finally {
|
||||
if (stopped) {
|
||||
String message = String.format("Fetching metadata for %s interrupted", post.getUrl());
|
||||
logEvent.setStatus(LogEvent.Status.DONE);
|
||||
logEvent.setMessage(message);
|
||||
eventRepository.update(logEvent);
|
||||
}
|
||||
finished = true;
|
||||
metadataService.stopFetchingMetadata(List.of(post.getPostId()));
|
||||
}
|
||||
}
|
||||
|
||||
private Metadata fetchMetadata(String postId, String threadId, String url) {
|
||||
HttpGet httpGet = cm.buildHttpGet(url, context);
|
||||
AtomicReference<Metadata> metadataReference = new AtomicReference<>();
|
||||
Failsafe.with(cm.getRetryPolicy())
|
||||
.onFailure(
|
||||
e ->
|
||||
log.error(
|
||||
String.format("Error occurred when getting post metadata, postId %s", postId),
|
||||
e.getFailure()))
|
||||
.run(
|
||||
() -> {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
HttpClient connection = cm.getClient().build();
|
||||
try (CloseableHttpResponse response =
|
||||
(CloseableHttpResponse) connection.execute(httpGet, context)) {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(
|
||||
String.format(
|
||||
"Unexpected response code '%d' for %s",
|
||||
response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
try {
|
||||
Document document =
|
||||
htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
Node postNode =
|
||||
xpathService.getAsNode(
|
||||
document,
|
||||
String.format(
|
||||
"//li[@id='post_%s']/div[contains(@class, 'postdetails')]", postId));
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
String postedBy =
|
||||
xpathService
|
||||
.getAsNode(
|
||||
postNode,
|
||||
"./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font")
|
||||
.getTextContent()
|
||||
.trim();
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostedBy(postedBy);
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
Node node =
|
||||
xpathService.getAsNode(
|
||||
document, String.format("//div[@id='post_message_%s']", postId));
|
||||
metadata.setResolvedNames(findTitleInContent(node));
|
||||
metadata.setPostId(postId);
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
metadataReference.set(metadata);
|
||||
} catch (Exception e) {
|
||||
if (stopped) {
|
||||
log.warn(e.getMessage(), e);
|
||||
return;
|
||||
}
|
||||
throw new PostParseException(
|
||||
String.format("Failed to parse thread %s, post %s", threadId, postId), e);
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
}
|
||||
}
|
||||
});
|
||||
return metadataReference.get();
|
||||
}
|
||||
|
||||
private List<String> findTitleInContent(Node node) {
|
||||
List<String> altTitle = new ArrayList<>();
|
||||
findTitle(node, altTitle, new AtomicBoolean(true));
|
||||
return altTitle.stream().distinct().collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private void findTitle(Node node, List<String> altTitle, AtomicBoolean keepGoing) {
|
||||
if (!keepGoing.get()) {
|
||||
return;
|
||||
}
|
||||
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
|
||||
keepGoing.set(false);
|
||||
return;
|
||||
}
|
||||
if (node.getNodeType() == Node.ELEMENT_NODE) {
|
||||
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
|
||||
Node item = node.getChildNodes().item(i);
|
||||
findTitle(item, altTitle, keepGoing);
|
||||
if (!keepGoing.get()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
} else if (node.getNodeType() == Node.TEXT_NODE) {
|
||||
String text = node.getTextContent().trim();
|
||||
if (!text.isBlank()
|
||||
&& dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
|
||||
altTitle.add(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
this.stopped = true;
|
||||
List<AbstractExecutionAwareRequest> requests =
|
||||
(List<AbstractExecutionAwareRequest>)
|
||||
this.context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
|
||||
if (requests != null) {
|
||||
for (AbstractExecutionAwareRequest request : requests) {
|
||||
request.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -6,16 +6,16 @@ import org.springframework.web.bind.annotation.CrossOrigin;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@CrossOrigin(value = "*")
|
||||
public class EventLogRestEndpoint {
|
||||
|
||||
private final IEventRepository eventRepository;
|
||||
private final ILogEventRepository eventRepository;
|
||||
|
||||
public EventLogRestEndpoint(IEventRepository eventRepository) {
|
||||
public EventLogRestEndpoint(ILogEventRepository eventRepository) {
|
||||
this.eventRepository = eventRepository;
|
||||
}
|
||||
|
||||
|
||||
+12
-10
@@ -9,14 +9,11 @@ import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.PathService;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.ThreadPoolService;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostItem;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.tasks.LinkScanRunnable;
|
||||
import tn.mnlr.vripper.web.restendpoints.domain.*;
|
||||
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
|
||||
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
|
||||
@@ -39,6 +36,7 @@ public class PostRestEndpoint {
|
||||
private final DownloadService downloadService;
|
||||
private final PostService postService;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final SettingsService settingsService;
|
||||
|
||||
@Autowired
|
||||
public PostRestEndpoint(
|
||||
@@ -46,12 +44,14 @@ public class PostRestEndpoint {
|
||||
PathService pathService,
|
||||
DownloadService downloadService,
|
||||
PostService postService,
|
||||
ThreadPoolService threadPoolService) {
|
||||
ThreadPoolService threadPoolService,
|
||||
SettingsService settingsService) {
|
||||
this.dataService = dataService;
|
||||
this.pathService = pathService;
|
||||
this.downloadService = downloadService;
|
||||
this.postService = postService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.settingsService = settingsService;
|
||||
}
|
||||
|
||||
@PostMapping("/post")
|
||||
@@ -108,7 +108,8 @@ public class PostRestEndpoint {
|
||||
log.error("Download has not been started yet for this post");
|
||||
throw new NotFoundException("Download has not been started yet for this post");
|
||||
} else {
|
||||
return new DownloadPath(pathService.calcDownloadDirectory(post).getPath());
|
||||
return new DownloadPath(
|
||||
pathService.calcDownloadDirectory(post, settingsService.getSettings()).getPath());
|
||||
}
|
||||
} else {
|
||||
log.error(String.format("Unable to find post with postId = %s", postId));
|
||||
@@ -169,7 +170,8 @@ public class PostRestEndpoint {
|
||||
synchronized (LOCK) {
|
||||
for (AltPostName altPostName : postToRename) {
|
||||
try {
|
||||
pathService.rename(altPostName.getPostId(), altPostName.getAltName());
|
||||
pathService.rename(
|
||||
altPostName.getPostId(), altPostName.getAltName(), settingsService.getSettings());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
String.format("Failed to rename post with postId = %s", altPostName.getPostId()), e);
|
||||
@@ -199,7 +201,7 @@ public class PostRestEndpoint {
|
||||
if (!resolvedNames.isEmpty()) {
|
||||
String altTitle = resolvedNames.get(0);
|
||||
try {
|
||||
pathService.rename(postId.getPostId(), altTitle);
|
||||
pathService.rename(postId.getPostId(), altTitle, settingsService.getSettings());
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
String.format("Failed to rename post with postId = %s", postId.getPostId()), e);
|
||||
|
||||
+3
-2
@@ -6,6 +6,7 @@ import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import tn.mnlr.vripper.exception.ValidationException;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
|
||||
|
||||
import java.util.List;
|
||||
@@ -37,7 +38,7 @@ public class SettingsRestEndpoint {
|
||||
|
||||
@PostMapping("/settings")
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public SettingsService.Settings postSettings(@RequestBody SettingsService.Settings settings) {
|
||||
public Settings postSettings(@RequestBody Settings settings) {
|
||||
|
||||
try {
|
||||
this.settingsService.check(settings);
|
||||
@@ -52,7 +53,7 @@ public class SettingsRestEndpoint {
|
||||
|
||||
@GetMapping("/settings")
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public SettingsService.Settings getAppSettingsService() {
|
||||
public Settings getAppSettingsService() {
|
||||
|
||||
return settingsService.getSettings();
|
||||
}
|
||||
|
||||
@@ -5,22 +5,19 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.Disposable;
|
||||
import tn.mnlr.vripper.event.*;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.listener.*;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.DownloadSpeedService;
|
||||
import tn.mnlr.vripper.services.GlobalStateService;
|
||||
import tn.mnlr.vripper.services.VGAuthService;
|
||||
import tn.mnlr.vripper.services.domain.DownloadSpeed;
|
||||
import tn.mnlr.vripper.services.domain.GlobalState;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.time.Duration;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@@ -29,204 +26,133 @@ import java.util.stream.Collectors;
|
||||
public class DataBroadcast {
|
||||
|
||||
private final SimpMessagingTemplate template;
|
||||
private final VGAuthService VGAuthService;
|
||||
private final GlobalStateService globalStateService;
|
||||
private final DownloadSpeedService downloadSpeedService;
|
||||
private final DataService dataService;
|
||||
private final EventBus eventBus;
|
||||
|
||||
private final PostUpdateEventListener postUpdateEventListener;
|
||||
private final MetadataUpdateEventListener metadataUpdateEventListener;
|
||||
private final ImageUpdateEventListener imageUpdateEventListener;
|
||||
private final QueuedUpdateEventListener queuedUpdateEventListener;
|
||||
private final EventUpdateEventListener eventUpdateEventListener;
|
||||
|
||||
private final QueuedRemoveEventListener queuedRemoveEventListener;
|
||||
private final PostRemoveEventListener postRemoveEventListener;
|
||||
private final EventRemoveEventListener eventRemoveEventListener;
|
||||
|
||||
private final List<Disposable> disposables = new ArrayList<>();
|
||||
private Disposable disposable;
|
||||
|
||||
@Autowired
|
||||
public DataBroadcast(
|
||||
SimpMessagingTemplate template,
|
||||
VGAuthService VGAuthService,
|
||||
GlobalStateService globalStateService,
|
||||
DownloadSpeedService downloadSpeedService,
|
||||
DataService dataService,
|
||||
PostUpdateEventListener postUpdateEventListener,
|
||||
MetadataUpdateEventListener metadataUpdateEventListener,
|
||||
ImageUpdateEventListener imageUpdateEventListener,
|
||||
QueuedUpdateEventListener queuedUpdateEventListener,
|
||||
EventUpdateEventListener eventUpdateEventListener,
|
||||
QueuedRemoveEventListener queuedRemoveEventListener,
|
||||
PostRemoveEventListener postRemoveEventListener,
|
||||
EventRemoveEventListener eventRemoveEventListener) {
|
||||
public DataBroadcast(SimpMessagingTemplate template, DataService dataService, EventBus eventBus) {
|
||||
this.template = template;
|
||||
this.VGAuthService = VGAuthService;
|
||||
this.globalStateService = globalStateService;
|
||||
this.downloadSpeedService = downloadSpeedService;
|
||||
this.dataService = dataService;
|
||||
this.postUpdateEventListener = postUpdateEventListener;
|
||||
this.metadataUpdateEventListener = metadataUpdateEventListener;
|
||||
this.imageUpdateEventListener = imageUpdateEventListener;
|
||||
this.queuedUpdateEventListener = queuedUpdateEventListener;
|
||||
this.eventUpdateEventListener = eventUpdateEventListener;
|
||||
this.queuedRemoveEventListener = queuedRemoveEventListener;
|
||||
this.postRemoveEventListener = postRemoveEventListener;
|
||||
this.eventRemoveEventListener = eventRemoveEventListener;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void run() {
|
||||
disposables.add(
|
||||
VGAuthService.getLoggedInUser()
|
||||
.map(DataController.LoggedUser::new)
|
||||
.subscribe(
|
||||
user -> template.convertAndSend("/topic/user", user),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
|
||||
disposables.add(
|
||||
globalStateService
|
||||
.getGlobalState()
|
||||
.subscribe(
|
||||
state -> template.convertAndSend("/topic/download-state", state),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
|
||||
disposables.add(
|
||||
downloadSpeedService
|
||||
.getReadBytesPerSecond()
|
||||
.map(DownloadSpeed::new)
|
||||
.subscribe(
|
||||
speed -> template.convertAndSend("/topic/speed", speed),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
|
||||
disposables.add(
|
||||
postUpdateEventListener
|
||||
.getDataFlux()
|
||||
.map(PostUpdateEvent::getId)
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
ids ->
|
||||
template.convertAndSend(
|
||||
"/topic/posts",
|
||||
ids.stream()
|
||||
.map(dataService::findById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toList())),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
data -> {
|
||||
Map<Event.Kind, List<Event<?>>> eventMap =
|
||||
data.stream().collect(Collectors.groupingBy(Event::getKind));
|
||||
|
||||
disposables.add(
|
||||
metadataUpdateEventListener
|
||||
.getDataFlux()
|
||||
.map(MetadataUpdateEvent::getPostIdRef)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
ids ->
|
||||
template.convertAndSend(
|
||||
"/topic/posts",
|
||||
ids.stream()
|
||||
.map(dataService::findById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toList())),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
eventMap.forEach(
|
||||
(kind, eventList) -> {
|
||||
switch (kind) {
|
||||
case POST_UPDATE:
|
||||
case METADATA_UPDATE:
|
||||
template.convertAndSend(
|
||||
"/topic/posts",
|
||||
eventList.stream()
|
||||
.map(e -> ((Long) e.getData()))
|
||||
.distinct()
|
||||
.map(dataService::findById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toUnmodifiableSet()));
|
||||
|
||||
disposables.add(
|
||||
eventUpdateEventListener
|
||||
.getDataFlux()
|
||||
.map(EventUpdateEvent::getId)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
ids ->
|
||||
template.convertAndSend(
|
||||
"/topic/events",
|
||||
ids.stream()
|
||||
.map(dataService::findEventById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toList())),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
break;
|
||||
case POST_REMOVE:
|
||||
template.convertAndSend(
|
||||
"/topic/posts/deleted",
|
||||
eventList.stream()
|
||||
.map(e -> ((String) e.getData()))
|
||||
.collect(Collectors.toUnmodifiableSet()));
|
||||
break;
|
||||
case IMAGE_UPDATE:
|
||||
eventList.stream()
|
||||
.map(e -> ((Long) e.getData()))
|
||||
.distinct()
|
||||
.map(dataService::findImageById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.groupingBy(Image::getPostId))
|
||||
.forEach(
|
||||
(postId, images) ->
|
||||
template.convertAndSend("/topic/images/" + postId, images));
|
||||
break;
|
||||
|
||||
disposables.add(
|
||||
imageUpdateEventListener
|
||||
.getDataFlux()
|
||||
.map(ImageUpdateEvent::getId)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
id ->
|
||||
id.stream()
|
||||
.map(dataService::findImageById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.groupingBy(Image::getPostId))
|
||||
.forEach(
|
||||
(postId, images) ->
|
||||
template.convertAndSend("/topic/images/" + postId, images)),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
case QUEUED_UPDATE:
|
||||
template.convertAndSend(
|
||||
"/topic/queued",
|
||||
eventList.stream()
|
||||
.map(e -> ((Long) e.getData()))
|
||||
.distinct()
|
||||
.map(dataService::findQueuedById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toUnmodifiableSet()));
|
||||
break;
|
||||
case QUEUED_REMOVE:
|
||||
template.convertAndSend(
|
||||
"/topic/queued/deleted",
|
||||
eventList.stream()
|
||||
.map(e -> ((String) e.getData()))
|
||||
.collect(Collectors.toUnmodifiableSet()));
|
||||
break;
|
||||
case LOG_EVENT_UPDATE:
|
||||
template.convertAndSend(
|
||||
"/topic/events",
|
||||
eventList.stream()
|
||||
.map(e -> ((Long) e.getData()))
|
||||
.distinct()
|
||||
.map(dataService::findEventById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toUnmodifiableSet()));
|
||||
break;
|
||||
case LOG_EVENT_REMOVE:
|
||||
template.convertAndSend(
|
||||
"/topic/events/deleted",
|
||||
eventList.stream()
|
||||
.map(e -> ((Long) e.getData()))
|
||||
.collect(Collectors.toUnmodifiableSet()));
|
||||
break;
|
||||
case VG_USER:
|
||||
eventList.stream()
|
||||
.map(e -> new DataController.LoggedUser((String) e.getData()))
|
||||
.distinct()
|
||||
.forEach(user -> template.convertAndSend("/topic/user", user));
|
||||
break;
|
||||
case GLOBAL_STATE:
|
||||
eventList.stream()
|
||||
.map(e -> ((GlobalState) e.getData()))
|
||||
.distinct()
|
||||
.forEach(
|
||||
globalState ->
|
||||
template.convertAndSend(
|
||||
"/topic/download-state", globalState));
|
||||
break;
|
||||
case BYTES_PER_SECOND:
|
||||
eventList.stream()
|
||||
.map(e -> new DownloadSpeed((Long) e.getData()))
|
||||
.distinct()
|
||||
.forEach(speed -> template.convertAndSend("/topic/speed", speed));
|
||||
|
||||
disposables.add(
|
||||
queuedUpdateEventListener
|
||||
.getDataFlux()
|
||||
.map(QueuedUpdateEvent::getId)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
ids ->
|
||||
template.convertAndSend(
|
||||
"/topic/queued",
|
||||
ids.stream()
|
||||
.map(dataService::findQueuedById)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toList())),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
|
||||
disposables.add(
|
||||
queuedRemoveEventListener
|
||||
.getDataFlux()
|
||||
.map(QueuedRemoveEvent::getThreadId)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
threadIds -> template.convertAndSend("/topic/queued/deleted", threadIds),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
|
||||
disposables.add(
|
||||
postRemoveEventListener
|
||||
.getDataFlux()
|
||||
.map(PostRemoveEvent::getPostId)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
postIds -> template.convertAndSend("/topic/posts/deleted", postIds),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
|
||||
disposables.add(
|
||||
eventRemoveEventListener
|
||||
.getDataFlux()
|
||||
.map(EventRemoveEvent::getId)
|
||||
.buffer(Duration.of(500, ChronoUnit.MILLIS))
|
||||
.map(HashSet::new)
|
||||
.filter(e -> !e.isEmpty())
|
||||
.subscribe(
|
||||
postIds -> template.convertAndSend("/topic/events/deleted", postIds),
|
||||
e -> log.error("Failed to send data to client", e)));
|
||||
break;
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
disposables.forEach(Disposable::dispose);
|
||||
if (disposable != null) {
|
||||
disposable.dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,8 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.messaging.handler.annotation.DestinationVariable;
|
||||
import org.springframework.messaging.simp.annotation.SubscribeMapping;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
@@ -17,6 +17,7 @@ import tn.mnlr.vripper.services.domain.DownloadSpeed;
|
||||
import tn.mnlr.vripper.services.domain.GlobalState;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
@Controller
|
||||
@@ -56,7 +57,9 @@ public class DataController {
|
||||
|
||||
@SubscribeMapping("/posts")
|
||||
public Collection<Post> posts() {
|
||||
return dataService.findAllPosts();
|
||||
List<Post> posts = dataService.findAllPosts();
|
||||
posts.sort(Comparator.comparing(Post::getAddedOn));
|
||||
return posts;
|
||||
}
|
||||
|
||||
@SubscribeMapping("/images/{postId}")
|
||||
@@ -70,7 +73,7 @@ public class DataController {
|
||||
}
|
||||
|
||||
@SubscribeMapping("/events")
|
||||
public Collection<Event> events() {
|
||||
public Collection<LogEvent> events() {
|
||||
return dataService.findAllEvents();
|
||||
}
|
||||
|
||||
|
||||
@@ -161,4 +161,18 @@
|
||||
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
|
||||
referencedColumnNames="ID" referencedTableName="POST"/>
|
||||
</changeSet>
|
||||
<changeSet id="1620590428" author="death-claw">
|
||||
<addColumn tableName="POST">
|
||||
<column name="ADDED_ON" type="TIMESTAMP" defaultValueComputed="CURRENT_TIMESTAMP">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
</addColumn>
|
||||
</changeSet>
|
||||
<changeSet id="1621412143" author="death-claw">
|
||||
<addColumn tableName="POST">
|
||||
<column name="RANK" type="INT" defaultValue="0">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
</addColumn>
|
||||
</changeSet>
|
||||
</databaseChangeLog>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-ui",
|
||||
"version": "3.3.8",
|
||||
"version": "3.5.0",
|
||||
"scripts": {
|
||||
"ng": "ng",
|
||||
"start": "ng serve",
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.8</version>
|
||||
<version>3.5.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-ui</artifactId>
|
||||
<name>vripper-ui</name>
|
||||
|
||||
@@ -11,7 +11,9 @@ export class Post {
|
||||
public hosts: string[],
|
||||
public thanked: boolean,
|
||||
public previews: string[],
|
||||
public metadata: Metadata
|
||||
public metadata: Metadata,
|
||||
public addedOn: string,
|
||||
public rank: number
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import {GridApi, ICellRendererComp, ICellRendererParams, RowNode} from 'ag-grid-community';
|
||||
import {PostContextMenuService} from '../services/post-context-menu.service';
|
||||
|
||||
export class PostAddedRendererNative implements ICellRendererComp {
|
||||
private gui: HTMLElement;
|
||||
private addedOn: string;
|
||||
private text: HTMLSpanElement;
|
||||
private gridApi: GridApi;
|
||||
private node: RowNode;
|
||||
private contextMenuService: PostContextMenuService;
|
||||
|
||||
destroy(): void {
|
||||
if (this.gui) {
|
||||
this.gui.removeEventListener('contextmenu', this.context.bind(this));
|
||||
}
|
||||
}
|
||||
|
||||
getGui(): HTMLElement {
|
||||
return this.gui;
|
||||
}
|
||||
|
||||
init(params: ICellRendererParams): void {
|
||||
// @ts-ignore
|
||||
this.contextMenuService = params.contextMenuService;
|
||||
this.gridApi = params.api;
|
||||
this.node = params.node;
|
||||
this.addedOn = params.node.data.addedOn;
|
||||
this.gui = document.createElement('div');
|
||||
this.gui.classList.add('text-cell');
|
||||
this.text = document.createElement('span');
|
||||
this.text.innerText = this.addedOn;
|
||||
this.text.setAttribute('title', this.addedOn);
|
||||
this.gui.append(this.text);
|
||||
this.gui.addEventListener('contextmenu', this.context.bind(this));
|
||||
}
|
||||
|
||||
refresh(params: ICellRendererParams): boolean {
|
||||
this.addedOn = params.node.data.addedOn;
|
||||
this.text.innerText = this.addedOn;
|
||||
this.text.setAttribute('title', this.addedOn);
|
||||
return true;
|
||||
}
|
||||
|
||||
context(event: MouseEvent) {
|
||||
event.preventDefault();
|
||||
this.gridApi.getSelectedNodes().forEach(e => e.setSelected(false));
|
||||
this.node.setSelected(true);
|
||||
this.contextMenuService.openPostContextMenu(event, this.node.data);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import {GridApi, ICellRendererComp, ICellRendererParams, RowNode} from 'ag-grid-community';
|
||||
import {PostContextMenuService} from '../services/post-context-menu.service';
|
||||
|
||||
export class PostOrderRendererNative implements ICellRendererComp {
|
||||
private gui: HTMLElement;
|
||||
private order: number;
|
||||
private text: HTMLSpanElement;
|
||||
private gridApi: GridApi;
|
||||
private node: RowNode;
|
||||
private contextMenuService: PostContextMenuService;
|
||||
|
||||
destroy(): void {
|
||||
if (this.gui) {
|
||||
this.gui.removeEventListener('contextmenu', this.context.bind(this));
|
||||
}
|
||||
}
|
||||
|
||||
getGui(): HTMLElement {
|
||||
return this.gui;
|
||||
}
|
||||
|
||||
init(params: ICellRendererParams): void {
|
||||
// @ts-ignore
|
||||
this.contextMenuService = params.contextMenuService;
|
||||
this.gridApi = params.api;
|
||||
this.node = params.node;
|
||||
this.order = params.node.data.rank;
|
||||
this.gui = document.createElement('div');
|
||||
this.gui.classList.add('text-cell');
|
||||
this.text = document.createElement('span');
|
||||
this.text.innerText = this.order.toString(10);
|
||||
this.text.setAttribute('title', this.order.toString(10));
|
||||
this.gui.append(this.text);
|
||||
this.gui.addEventListener('contextmenu', this.context.bind(this));
|
||||
}
|
||||
|
||||
refresh(params: ICellRendererParams): boolean {
|
||||
this.order = params.node.data.rank;
|
||||
this.text.innerText = this.order.toString(10);
|
||||
this.text.setAttribute('title', this.order.toString(10));
|
||||
return true;
|
||||
}
|
||||
|
||||
context(event: MouseEvent) {
|
||||
event.preventDefault();
|
||||
this.gridApi.getSelectedNodes().forEach(e => e.setSelected(false));
|
||||
this.node.setSelected(true);
|
||||
this.contextMenuService.openPostContextMenu(event, this.node.data);
|
||||
}
|
||||
}
|
||||
@@ -85,7 +85,7 @@ export class PostContextMenuComponent {
|
||||
maxHeight: '100vh',
|
||||
maxWidth: '100vw',
|
||||
height: '200px',
|
||||
width: '60%',
|
||||
width: '400px',
|
||||
data: {header: 'Confirmation', content: 'Are you sure you want to remove this item ?'}
|
||||
})
|
||||
.afterClosed()
|
||||
|
||||
@@ -11,6 +11,8 @@ import {PostAltRendererNative} from '../grid-custom-cells/post-alt-renderer.nati
|
||||
import {TitleRendererNative} from '../grid-custom-cells/title-renderer.native';
|
||||
import {Overlay, OverlayPositionBuilder} from '@angular/cdk/overlay';
|
||||
import {PostsService} from '../services/posts.service';
|
||||
import {PostAddedRendererNative} from '../grid-custom-cells/post-added-renderer.native';
|
||||
import {PostOrderRendererNative} from '../grid-custom-cells/post-order-renderer.native';
|
||||
|
||||
@Component({
|
||||
selector: 'app-posts',
|
||||
@@ -43,7 +45,6 @@ export class PostsComponent implements OnDestroy {
|
||||
overlay: this.overlay,
|
||||
zone: this.zone
|
||||
},
|
||||
sort: 'asc',
|
||||
headerCheckboxSelection: true,
|
||||
headerCheckboxSelectionFilteredOnly: true,
|
||||
flex: 2
|
||||
@@ -81,6 +82,24 @@ export class PostsComponent implements OnDestroy {
|
||||
contextMenuService: this.contextMenuService
|
||||
},
|
||||
flex: 1
|
||||
}, {
|
||||
headerName: 'Added On',
|
||||
field: 'addedOn',
|
||||
cellRenderer: 'nativeAddedCellRenderer',
|
||||
cellRendererParams: {
|
||||
contextMenuService: this.contextMenuService
|
||||
},
|
||||
flex: 1
|
||||
}, {
|
||||
headerName: 'Order',
|
||||
field: 'rank',
|
||||
cellRenderer: 'nativeOrderCellRenderer',
|
||||
cellRendererParams: {
|
||||
contextMenuService: this.contextMenuService,
|
||||
},
|
||||
width: 100,
|
||||
maxWidth: 150,
|
||||
sort: 'asc'
|
||||
}
|
||||
],
|
||||
defaultColDef: {
|
||||
@@ -99,6 +118,8 @@ export class PostsComponent implements OnDestroy {
|
||||
nativeFilesCellRenderer: PostFilesRendererNative,
|
||||
nativeAltCellRenderer: PostAltRendererNative,
|
||||
nativeTitleCellRenderer: TitleRendererNative,
|
||||
nativeAddedCellRenderer: PostAddedRendererNative,
|
||||
nativeOrderCellRenderer: PostOrderRendererNative,
|
||||
},
|
||||
overlayLoadingTemplate: '<span></span>',
|
||||
overlayNoRowsTemplate: '<span></span>',
|
||||
|
||||
@@ -82,7 +82,7 @@ export class WsConnectionService {
|
||||
if (this.electronService.isElectronApp) {
|
||||
const portRequest = setInterval(() => {
|
||||
this.electronService.ipcRenderer.send('get-port');
|
||||
}, 1000);
|
||||
}, 200);
|
||||
|
||||
// wait for MainIPC
|
||||
this.electronService.ipcRenderer.once('port', (event, port) => {
|
||||
@@ -107,7 +107,7 @@ export class WsConnectionService {
|
||||
Authorization: localStorage.getItem('auth') ? 'Bearer ' + localStorage.getItem('auth') : ''
|
||||
},
|
||||
brokerURL: this.serverService.wsBaseUrl + '/ws',
|
||||
reconnectDelay: 5000,
|
||||
reconnectDelay: 200,
|
||||
// debug: function (str) {
|
||||
// console.log('STOMP: ' + str);
|
||||
// },
|
||||
@@ -171,7 +171,7 @@ export class WsConnectionService {
|
||||
this.posts = this.rxStomp.watch('/topic/posts').pipe(
|
||||
map(e => {
|
||||
const posts: Array<Post> = [];
|
||||
(<Array<any>>JSON.parse(e.body)).forEach(element => {
|
||||
(<Array<any>>JSON.parse(e.body)).forEach((element, index) => {
|
||||
posts.push(
|
||||
new Post(
|
||||
element.postId,
|
||||
@@ -185,7 +185,9 @@ export class WsConnectionService {
|
||||
element.hosts,
|
||||
element.thanked,
|
||||
element.previews,
|
||||
element.metadata
|
||||
element.metadata,
|
||||
element.addedOn,
|
||||
element.rank + 1
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
@@ -105,7 +105,7 @@ export class ToolbarComponent implements OnInit, OnDestroy {
|
||||
maxHeight: '100vh',
|
||||
maxWidth: '100vw',
|
||||
height: '200px',
|
||||
width: '60%',
|
||||
width: '400px',
|
||||
data: {header: 'Confirmation', content: 'Are you sure you want to remove the selected items ?'}
|
||||
})
|
||||
.afterClosed()
|
||||
|
||||
@@ -2,5 +2,5 @@ export const environment = {
|
||||
production: true,
|
||||
localhost: `${window.location.protocol}//${window.location.host}`,
|
||||
ws: `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}`,
|
||||
version: '3.3.8'
|
||||
version: '3.5.0'
|
||||
};
|
||||
|
||||
@@ -6,7 +6,7 @@ export const environment = {
|
||||
production: false,
|
||||
localhost: 'http://localhost:8080',
|
||||
ws: 'ws://localhost:8080',
|
||||
version: '3.3.8'
|
||||
version: '3.5.0'
|
||||
};
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user