mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1247faacc | ||
|
|
efcc685f30 | ||
|
|
c825b22ce4 | ||
|
|
6ef5d5474f | ||
|
|
bd25801c6c | ||
|
|
377b93c1ac | ||
|
|
d498d20f6b | ||
|
|
d18544d414 | ||
|
|
3c872b79bd | ||
|
|
37900cac49 | ||
|
|
be57c76d93 |
@@ -1,5 +1,13 @@
|
||||
# Changelog
|
||||
|
||||
## [3.0.3] - 2020-08-16
|
||||
### Changed
|
||||
- Fix bugs with electron app
|
||||
|
||||
## [3.0.2] - 2020-08-15
|
||||
### Changed
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.1] - 2020-08-03
|
||||
### Changed
|
||||
- Fix bugs with download queue
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>3.0.3</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
+63
-50
@@ -1,13 +1,16 @@
|
||||
require('v8-compile-cache');
|
||||
const { app, BrowserWindow } = require("electron");
|
||||
const {app, BrowserWindow} = require("electron");
|
||||
const path = require("path");
|
||||
const url = require("url");
|
||||
const getPort = require("get-port");
|
||||
const { spawn } = require("child_process");
|
||||
const { ipcMain } = require("electron");
|
||||
const { dialog } = require("electron");
|
||||
const {spawn} = require("child_process");
|
||||
const {ipcMain} = require("electron");
|
||||
const {dialog} = require("electron");
|
||||
const axios = require('axios');
|
||||
const appDir = process.env.APPDIR;
|
||||
|
||||
// non null value when it is an AppImage
|
||||
const appImageDir = process.env.APPDIR;
|
||||
const appImagePath = process.env.APPIMAGE;
|
||||
|
||||
let win;
|
||||
let vripperServer;
|
||||
@@ -23,14 +26,14 @@ process.on("uncaughtException", err => {
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
function createWindow() {
|
||||
createWindow = () => {
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
app.setAppUserModelId("tn.mnlr.vripper");
|
||||
}
|
||||
|
||||
let icon;
|
||||
if(process.platform === "win32") {
|
||||
if (process.platform === "win32") {
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
icon = __dirname + '/icon.png';
|
||||
@@ -66,6 +69,33 @@ function createWindow() {
|
||||
});
|
||||
}
|
||||
|
||||
shutdownServer = () => {
|
||||
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
|
||||
headers: {'content-type': 'application/json'},
|
||||
}).then((response) => {
|
||||
terminationInteval = setInterval(() => {
|
||||
terminationAttemps++;
|
||||
if (terminated) {
|
||||
console.log('viper server terminated');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
} else if (terminationAttemps > maxTerminationAttemps) {
|
||||
console.log('viper server is not terminated');
|
||||
console.log('Proceed to kill');
|
||||
vripperServer.kill('SIGKILL');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
}
|
||||
}, 1000);
|
||||
}).catch((error) => {
|
||||
// Terminate immediately
|
||||
console.log(error);
|
||||
vripperServer.kill('SIGKILL');
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
}
|
||||
|
||||
const gotTheLock = app.requestSingleInstanceLock();
|
||||
|
||||
if (!gotTheLock) {
|
||||
@@ -76,29 +106,30 @@ if (!gotTheLock) {
|
||||
ipcMain.on("get-port", event => {
|
||||
event.reply("port", port);
|
||||
});
|
||||
let javaBinPath;
|
||||
if(appDir !== undefined) {
|
||||
javaBinPath = path.join(appDir, "java-runtime/bin/java");
|
||||
|
||||
const appPath = path.join(app.getAppPath(), '../../');
|
||||
let javaBinPath, jarPath, baseDir;
|
||||
if (appImageDir !== undefined && process.platform === 'linux') {
|
||||
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
|
||||
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
|
||||
baseDir = path.join(appImagePath, '..');
|
||||
} else if (process.platform === 'darwin') {
|
||||
javaBinPath = path.join(appPath, "java-runtime/bin/java");
|
||||
jarPath = path.join(appPath, "bin/vripper-server.jar");
|
||||
baseDir = path.join(appPath, '../..');
|
||||
} else if (process.platform === 'win32') {
|
||||
javaBinPath = path.join(appPath, "java-runtime/bin/java");
|
||||
jarPath = path.join(appPath, "bin/vripper-server.jar");
|
||||
baseDir = appPath;
|
||||
} else {
|
||||
if(process.platform === 'darwin') {
|
||||
javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
|
||||
} else {
|
||||
javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
|
||||
}
|
||||
}
|
||||
let jarPath;
|
||||
if(appDir !== undefined) {
|
||||
jarPath = path.join(appDir, "bin/vripper-server.jar");
|
||||
} else {
|
||||
if(process.platform === 'darwin') {
|
||||
jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
|
||||
} else {
|
||||
jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
|
||||
}
|
||||
console.error(`Unknown platform ${process.platform}`);
|
||||
app.quit();
|
||||
}
|
||||
|
||||
vripperServer = spawn(javaBinPath, [
|
||||
"-Xms256m",
|
||||
"-Dvripper.server.port=" + port,
|
||||
"-Dbase.dir=" + baseDir,
|
||||
"-jar",
|
||||
jarPath
|
||||
], {
|
||||
@@ -122,31 +153,13 @@ if (!gotTheLock) {
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
if (process.platform !== "darwin") {
|
||||
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}).then((response) => {
|
||||
terminationInteval= setInterval(() => {
|
||||
terminationAttemps++;
|
||||
if(terminated) {
|
||||
console.log('viper server terminated');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
} else if(terminationAttemps > maxTerminationAttemps) {
|
||||
console.log('viper server is not terminated');
|
||||
console.log('Proceed to kill');
|
||||
vripperServer.kill('SIGKILL');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
}
|
||||
}, 1000);
|
||||
})
|
||||
.catch((error) => {
|
||||
// Terminate immediately
|
||||
console.log(error);
|
||||
vripperServer.kill('SIGKILL');
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
shutdownServer();
|
||||
}
|
||||
});
|
||||
|
||||
app.on("will-quit", () => {
|
||||
if (process.platform === "darwin") {
|
||||
shutdownServer();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Generated
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-electron",
|
||||
"version": "3.0.1",
|
||||
"version": "3.0.3",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-electron",
|
||||
"version": "3.0.1",
|
||||
"version": "3.0.3",
|
||||
"description": "A ripper for vipergirls.to built using web technolgies",
|
||||
"main": "main.js",
|
||||
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
|
||||
@@ -40,14 +40,9 @@
|
||||
"win": {
|
||||
"icon": "icon.ico",
|
||||
"target": [
|
||||
"nsis"
|
||||
"dir"
|
||||
]
|
||||
},
|
||||
"nsis": {
|
||||
"createDesktopShortcut": "always",
|
||||
"oneClick": false,
|
||||
"perMachine": false
|
||||
},
|
||||
"linux": {
|
||||
"synopsis": "vipergirls.to ripper",
|
||||
"category": "Utility",
|
||||
@@ -60,7 +55,7 @@
|
||||
"mac": {
|
||||
"category": "public.app-category.utilities",
|
||||
"target": [
|
||||
"dmg"
|
||||
"dir"
|
||||
],
|
||||
"icon": "icon.icns"
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>3.0.3</version>
|
||||
</parent>
|
||||
<artifactId>vripper-electron</artifactId>
|
||||
<name>vripper-electron</name>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>3.0.3</version>
|
||||
</parent>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<name>vripper-server</name>
|
||||
|
||||
@@ -6,17 +6,14 @@ import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Collections;
|
||||
|
||||
@SpringBootApplication
|
||||
@Slf4j
|
||||
public class VripperApplication {
|
||||
|
||||
public static final RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
|
||||
.handleIf(e -> !(e instanceof InterruptedException))
|
||||
.withDelay(1, 3, ChronoUnit.SECONDS)
|
||||
.withMaxAttempts(5)
|
||||
.abortOn(Collections.singletonList(InterruptedException.class))
|
||||
.onFailedAttempt(e -> log.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
|
||||
|
||||
public static void main(String[] args) {
|
||||
@@ -27,6 +24,5 @@ public class VripperApplication {
|
||||
log.error("Failed to run the application", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.q.DownloadJob;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
|
||||
@@ -79,17 +80,18 @@ abstract public class Host {
|
||||
return url.contains(getLookup());
|
||||
}
|
||||
|
||||
public void download(final Post post, final Image image, final ImageFileData imageFileData) throws DownloadException, InterruptedException {
|
||||
public void download(final Post post, final Image image, final ImageFileData imageFileData, DownloadJob downloadJob) throws DownloadException, InterruptedException {
|
||||
|
||||
image.setStatus(Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
|
||||
HttpClientContext context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
try {
|
||||
|
||||
image.setStatus(Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
|
||||
HttpClientContext context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
|
||||
synchronized (LOCK) {
|
||||
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
|
||||
post.setStatus(Status.DOWNLOADING);
|
||||
@@ -127,6 +129,9 @@ abstract public class Host {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
if (downloadJob.isStopped()) {
|
||||
return;
|
||||
}
|
||||
File destinationFolder;
|
||||
synchronized (LOCK) {
|
||||
Post updatedPost = dataService.findPostById(post.getId()).orElseThrow();
|
||||
@@ -138,6 +143,11 @@ abstract public class Host {
|
||||
}
|
||||
File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
|
||||
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
|
||||
if (downloadJob.isStopped()) {
|
||||
return;
|
||||
}
|
||||
|
||||
image.setTotal(response.getEntity().getContentLength());
|
||||
dataService.updateImageTotal(image.getTotal(), image.getId());
|
||||
|
||||
@@ -146,7 +156,7 @@ abstract public class Host {
|
||||
|
||||
byte[] buffer = new byte[READ_BUFFER_SIZE];
|
||||
int read;
|
||||
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
|
||||
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !downloadJob.isStopped()) {
|
||||
fos.write(buffer, 0, read);
|
||||
image.increase(read);
|
||||
downloadSpeedService.increase(read);
|
||||
@@ -154,22 +164,25 @@ abstract public class Host {
|
||||
}
|
||||
fos.flush();
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} finally {
|
||||
if (image.getCurrent() == image.getTotal()) {
|
||||
image.setStatus(Status.COMPLETE);
|
||||
} else {
|
||||
image.setStatus(Status.ERROR);
|
||||
if (downloadJob.isStopped()) {
|
||||
return;
|
||||
}
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
}
|
||||
File finalName = checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, imageFileData.getImageName(), image.getIndex());
|
||||
imageFileData.setFileName(finalName.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (Thread.interrupted() || imageFileData.getImageRequest().isAborted()) {
|
||||
throw new InterruptedException("Download was interrupted");
|
||||
}
|
||||
throw new DownloadException(e);
|
||||
} finally {
|
||||
if (image.getCurrent() == image.getTotal()) {
|
||||
image.setStatus(Status.COMPLETE);
|
||||
} else if (downloadJob.isStopped()) {
|
||||
image.setStatus(Status.STOPPED);
|
||||
} else {
|
||||
image.setStatus(Status.ERROR);
|
||||
}
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
downloadJob.done();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,9 +27,9 @@ public class Management {
|
||||
private final String backupFolder;
|
||||
|
||||
@Autowired
|
||||
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir) {
|
||||
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.backupFolder = baseDir + File.separator + ".vripper" + File.separator + "backup";
|
||||
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
|
||||
@@ -20,6 +20,12 @@ public class DownloadJob implements CheckedRunnable {
|
||||
@Getter
|
||||
private final ImageFileData imageFileData = new ImageFileData();
|
||||
|
||||
@Getter
|
||||
private boolean stopped = false;
|
||||
|
||||
@Getter
|
||||
private boolean finished = false;
|
||||
|
||||
DownloadJob(Post post, Image image) {
|
||||
this.image = image;
|
||||
this.post = post;
|
||||
@@ -27,9 +33,12 @@ public class DownloadJob implements CheckedRunnable {
|
||||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
if (stopped) {
|
||||
done();
|
||||
return;
|
||||
}
|
||||
log.debug(String.format("Starting downloading %s", image.getUrl()));
|
||||
image.getHost().download(post, image, imageFileData);
|
||||
|
||||
image.getHost().download(post, image, imageFileData, this);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -45,4 +54,12 @@ public class DownloadJob implements CheckedRunnable {
|
||||
public int hashCode() {
|
||||
return Objects.hash(image, post);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
public void done() {
|
||||
finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,25 +28,20 @@ public class ExecuteRunnable implements Runnable {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
mutexService.createPostLock(downloadJob.getPost().getPostId());
|
||||
ReentrantLock mutex = mutexService.getPostLock(downloadJob.getPost().getPostId());
|
||||
mutex.lock();
|
||||
Failsafe.with(VripperApplication.retryPolicy)
|
||||
.onFailure(e -> {
|
||||
if (e.getFailure() instanceof InterruptedException || e.getFailure().getCause() instanceof InterruptedException) {
|
||||
log.debug("Job successfully interrupted");
|
||||
return;
|
||||
}
|
||||
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
|
||||
downloadJob.getImage().setStatus(Status.ERROR);
|
||||
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
|
||||
})
|
||||
.onComplete(e -> {
|
||||
mutex.unlock();
|
||||
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
|
||||
executionService.afterJobFinish(downloadJob);
|
||||
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
|
||||
mutex.unlock();
|
||||
}).run(downloadJob);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,7 @@ import tn.mnlr.vripper.services.post.PostService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@@ -31,8 +29,7 @@ public class ExecutionService {
|
||||
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
|
||||
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
|
||||
private final Map<DownloadJob, Future<?>> futures = new ConcurrentHashMap<>();
|
||||
// private final Map<String, AtomicInteger> downloadCount = new ConcurrentHashMap<>();
|
||||
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
private final PendingQ pendingQ;
|
||||
private final AppSettingsService settings;
|
||||
@@ -75,18 +72,25 @@ public class ExecutionService {
|
||||
}
|
||||
|
||||
private void stopRunning(@NonNull String postId) {
|
||||
futures
|
||||
.entrySet()
|
||||
.stream()
|
||||
.filter(e -> e.getKey().getImage().getPostId().equals(postId))
|
||||
.forEach(e -> {
|
||||
e.getValue().cancel(true);
|
||||
if (e.getKey().getImageFileData().getImageRequest() != null) {
|
||||
e.getKey().getImageFileData().getImageRequest().abort();
|
||||
}
|
||||
e.getKey().getImage().setStatus(Status.STOPPED);
|
||||
dataService.updateImageStatus(e.getKey().getImage().getStatus(), e.getKey().getImage().getId());
|
||||
});
|
||||
List<DownloadJob> stopping = new ArrayList<>();
|
||||
Iterator<DownloadJob> iterator = executing.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob downloadJob = iterator.next();
|
||||
if (postId.equals(downloadJob.getPost().getPostId())) {
|
||||
downloadJob.stop();
|
||||
iterator.remove();
|
||||
stopping.add(downloadJob);
|
||||
}
|
||||
}
|
||||
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(DownloadJob::isFinished);
|
||||
try {
|
||||
Thread.sleep(200);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void stopAll(List<String> posIds) {
|
||||
@@ -142,9 +146,10 @@ public class ExecutionService {
|
||||
if (FINISHED.contains(post.getStatus())) {
|
||||
return;
|
||||
}
|
||||
pendingQ.remove(post);
|
||||
pendingQ.stop(post);
|
||||
stopRunning(postId);
|
||||
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
|
||||
dataService.finishPost(post);
|
||||
postService.stopFetchingMetadata(post);
|
||||
} finally {
|
||||
pauseQ = false;
|
||||
@@ -200,25 +205,25 @@ public class ExecutionService {
|
||||
}
|
||||
|
||||
private void push(DownloadJob downloadJob) {
|
||||
ExecuteRunnable runnable = new ExecuteRunnable(downloadJob);
|
||||
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
|
||||
futures.put(downloadJob, executor.submit(runnable));
|
||||
executor.execute(new ExecuteRunnable(downloadJob));
|
||||
executing.add(downloadJob);
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(DownloadJob downloadJob) {
|
||||
int count = pendingQ.afterJobFinish(downloadJob.getPost().getPostId());
|
||||
int count = pendingQ.decrement(downloadJob.getPost().getPostId());
|
||||
if (count == 0) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
mutexService.removePostLock(downloadJob.getPost().getPostId());
|
||||
}
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
futures.remove(downloadJob);
|
||||
executing.remove(downloadJob);
|
||||
synchronized (threadCount) {
|
||||
threadCount.notify();
|
||||
}
|
||||
}
|
||||
|
||||
public int runningCount() {
|
||||
return futures.size();
|
||||
return executing.size();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import java.util.concurrent.BlockingDeque;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -87,19 +88,25 @@ public class PendingQ {
|
||||
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
}
|
||||
|
||||
public void remove(Post post) {
|
||||
public void stop(Post post) {
|
||||
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
|
||||
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
|
||||
entry.getValue().removeIf(next -> next.getImage().getPostId().equals(post.getPostId()));
|
||||
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
|
||||
entry.getValue().removeIf(predicate);
|
||||
decrement(post.getPostId());
|
||||
}
|
||||
dataService.finishPost(post);
|
||||
}
|
||||
|
||||
public boolean isPending(String postId) {
|
||||
return toBeExecuted.containsKey(postId);
|
||||
}
|
||||
|
||||
public int afterJobFinish(String postId) {
|
||||
int count = toBeExecuted.get(postId).decrementAndGet();
|
||||
public synchronized int decrement(String postId) {
|
||||
AtomicInteger counter = toBeExecuted.get(postId);
|
||||
if (counter == null) {
|
||||
return 0;
|
||||
}
|
||||
int count = counter.decrementAndGet();
|
||||
if (count == 0) {
|
||||
toBeExecuted.remove(postId);
|
||||
}
|
||||
|
||||
@@ -38,9 +38,9 @@ public class AppSettingsService {
|
||||
|
||||
private Settings settings = new Settings();
|
||||
|
||||
public AppSettingsService(@Value("${base.dir}") String baseDir) {
|
||||
public AppSettingsService(@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
|
||||
this.baseDir = baseDir;
|
||||
this.configPath = Paths.get(baseDir, ".vripper", "config.json");
|
||||
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
|
||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
}
|
||||
|
||||
@@ -80,7 +80,7 @@ public class AppSettingsService {
|
||||
}
|
||||
|
||||
if (settings.getDownloadPath() == null) {
|
||||
settings.setDownloadPath(baseDir);
|
||||
settings.setDownloadPath(System.getProperty("user.home"));
|
||||
}
|
||||
|
||||
if (settings.getMaxThreads() == null) {
|
||||
@@ -139,11 +139,6 @@ public class AppSettingsService {
|
||||
settings.setViewPhotos(false);
|
||||
}
|
||||
|
||||
if (settings.getNotification() == null) {
|
||||
settings.setNotification(false);
|
||||
}
|
||||
|
||||
|
||||
save();
|
||||
}
|
||||
|
||||
@@ -237,8 +232,6 @@ public class AppSettingsService {
|
||||
private Boolean clearCompleted;
|
||||
@JsonProperty("viewPhotos")
|
||||
private Boolean viewPhotos;
|
||||
@JsonProperty("notification")
|
||||
private Boolean notification;
|
||||
@JsonProperty("darkTheme")
|
||||
private Boolean darkTheme;
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
@@ -79,7 +80,7 @@ public class PathService {
|
||||
post.setPostFolderName(newDestFolder.getName());
|
||||
dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
|
||||
|
||||
List<File> files = Optional.ofNullable(currentDesFolder.listFiles()).stream().flatMap(Arrays::stream).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
|
||||
List<File> files = Arrays.stream(Objects.requireNonNull(currentDesFolder.listFiles())).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
|
||||
for (File f : files) {
|
||||
try {
|
||||
Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||
@@ -89,6 +90,12 @@ public class PathService {
|
||||
}
|
||||
}
|
||||
|
||||
for (File file : Objects.requireNonNull(currentDesFolder.listFiles())) {
|
||||
if (!file.delete()) {
|
||||
log.warn(String.format("Failed to remove %s", file.toString()));
|
||||
}
|
||||
}
|
||||
|
||||
if (!currentDesFolder.delete()) {
|
||||
log.warn(String.format("Failed to remove %s", currentDesFolder.toString()));
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
base.dir=${user.home}
|
||||
@@ -1 +0,0 @@
|
||||
base.dir=${user.dir}
|
||||
@@ -2,15 +2,15 @@ logging.level.org.springframework.web=INFO
|
||||
logging.level.root=INFO
|
||||
logging.level.org.apache.http=INFO
|
||||
logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats=ERROR
|
||||
logging.file.name=${base.dir}/.vripper/vripper.log
|
||||
logging.file.name=${base.dir}/${base.dir.name}/vripper.log
|
||||
server.port=${vripper.server.port:8080}
|
||||
management.endpoints.web.exposure.include=shutdown
|
||||
management.endpoint.shutdown.enabled=true
|
||||
server.error.include-message=always
|
||||
spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.xml
|
||||
spring.datasource.url=jdbc:hsqldb:file:${base.dir}/.vripper/db/xparty
|
||||
spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/xparty;hsqldb.lock_file=false
|
||||
spring.datasource.username=SA
|
||||
spring.datasource.password=lEtmEIn
|
||||
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
|
||||
spring.profiles.active=portable
|
||||
#spring.profiles.active=installer
|
||||
base.dir=${user.dir}
|
||||
base.dir.name=${vripper.base.dir.name:vripper}
|
||||
|
||||
Generated
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-ui",
|
||||
"version": "3.0.1",
|
||||
"version": "3.0.3",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-ui",
|
||||
"version": "3.0.1",
|
||||
"version": "3.0.3",
|
||||
"scripts": {
|
||||
"ng": "ng",
|
||||
"start": "ng serve",
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<version>3.0.3</version>
|
||||
</parent>
|
||||
<artifactId>vripper-ui</artifactId>
|
||||
<name>vripper-ui</name>
|
||||
|
||||
@@ -38,7 +38,7 @@ export class ClipboardService {
|
||||
|
||||
_init(settings: Settings) {
|
||||
if (!this.electronService.isElectronApp) {
|
||||
console.log('Clipboard deactive, not an electron app');
|
||||
console.log('Clipboard deactivated, not an electron app');
|
||||
return;
|
||||
}
|
||||
if (this.interval != null) {
|
||||
|
||||
@@ -9,6 +9,5 @@ export interface Settings {
|
||||
vThanks: boolean;
|
||||
desktopClipboard: boolean;
|
||||
viewPhotos: boolean;
|
||||
notification: boolean;
|
||||
resolveTitle: boolean;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import {LinkCollectorService} from './../link-collector.service';
|
||||
import {LinkCollectorService} from '../link-collector.service';
|
||||
import {UrlGrabRendererComponent} from './renderer/url-renderer.component';
|
||||
import {GrabQueueDataSource} from './grab-queue.datasource';
|
||||
import {ChangeDetectionStrategy, Component, NgZone, OnInit} from '@angular/core';
|
||||
import {GridOptions} from 'ag-grid-community';
|
||||
import {WsConnectionService} from '../ws-connection.service';
|
||||
import {NotificationService} from '../notification.service';
|
||||
|
||||
@Component({
|
||||
selector: 'app-grab-queue',
|
||||
@@ -16,8 +15,7 @@ export class GrabQueueComponent implements OnInit {
|
||||
constructor(
|
||||
private wsConnection: WsConnectionService,
|
||||
private zone: NgZone,
|
||||
private linkCollectorService: LinkCollectorService,
|
||||
private notificationService: NotificationService) {
|
||||
private linkCollectorService: LinkCollectorService) {
|
||||
this.gridOptions = <GridOptions>{
|
||||
columnDefs: [
|
||||
{
|
||||
@@ -40,7 +38,7 @@ export class GrabQueueComponent implements OnInit {
|
||||
getRowNodeId: data => data['threadId'],
|
||||
onGridReady: () => {
|
||||
this.gridOptions.api.sizeColumnsToFit();
|
||||
this.dataSource = new GrabQueueDataSource(this.wsConnection, this.gridOptions, this.zone, this.notificationService);
|
||||
this.dataSource = new GrabQueueDataSource(this.wsConnection, this.gridOptions, this.zone);
|
||||
this.dataSource.connect();
|
||||
},
|
||||
onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {GrabQueueState} from '../domain/grab-queue.model';
|
||||
import {Subscription} from 'rxjs';
|
||||
import {WsConnectionService} from '../ws-connection.service';
|
||||
import {NotificationService} from '../notification.service';
|
||||
import {GridOptions, RowNode} from 'ag-grid-community';
|
||||
import {NgZone} from '@angular/core';
|
||||
|
||||
@@ -9,8 +8,7 @@ export class GrabQueueDataSource {
|
||||
constructor(
|
||||
private ws: WsConnectionService,
|
||||
private gridOptions: GridOptions,
|
||||
private zone: NgZone,
|
||||
private notificationService: NotificationService
|
||||
private zone: NgZone
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -30,13 +28,6 @@ export class GrabQueueDataSource {
|
||||
}
|
||||
});
|
||||
this.gridOptions.api.applyTransaction({update: toUpdate, add: toAdd});
|
||||
const count = this.gridOptions.api.getDisplayedRowCount();
|
||||
if (count > 0 && toAdd.length > 0) {
|
||||
this.notificationService.notifyFromGrabQueue(
|
||||
'Link Collector',
|
||||
`You have ${count} ${count > 1 ? 'threads' : 'thread'} waiting in the link collector`
|
||||
);
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
})
|
||||
export class NotificationService {
|
||||
constructor(private appService: AppService) {}
|
||||
|
||||
notifyFromGrabQueue(title: string, body: string) {
|
||||
if (!this.appService.settings.notification) {
|
||||
return;
|
||||
}
|
||||
if (!('Notification' in window)) {
|
||||
return;
|
||||
} else if (Notification.permission === 'granted') {
|
||||
const notification = new Notification(title, { body: body, icon: 'assets/icon.png' });
|
||||
} else if (Notification.permission !== 'denied') {
|
||||
Notification.requestPermission().then(function(permission) {
|
||||
if (permission === 'granted') {
|
||||
const notification = new Notification(title, { body: body, icon: 'assets/icon.png' });
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -110,11 +110,6 @@
|
||||
>Monitor Clipboard
|
||||
</mat-checkbox
|
||||
>
|
||||
|
||||
<mat-checkbox color="primary" formControlName="notification" name="notification"
|
||||
>Enable system notifications
|
||||
</mat-checkbox
|
||||
>
|
||||
</form>
|
||||
</mat-tab>
|
||||
</mat-tab-group>
|
||||
|
||||
@@ -46,8 +46,7 @@ export class SettingsComponent implements OnInit {
|
||||
});
|
||||
|
||||
desktopSettingsForm = new FormGroup({
|
||||
desktopClipboard: new FormControl(false),
|
||||
notification: new FormControl(false)
|
||||
desktopClipboard: new FormControl(false)
|
||||
});
|
||||
|
||||
darkTheme = false;
|
||||
|
||||
@@ -2,5 +2,5 @@ export const environment = {
|
||||
production: true,
|
||||
localhost: `${window.location.protocol}//${window.location.host}`,
|
||||
ws: `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}`,
|
||||
version: '3.0.1'
|
||||
version: '3.0.3'
|
||||
};
|
||||
|
||||
@@ -6,7 +6,7 @@ export const environment = {
|
||||
production: false,
|
||||
localhost: 'http://localhost:8080',
|
||||
ws: 'ws://localhost:8080',
|
||||
version: '3.0.1'
|
||||
version: '3.0.3'
|
||||
};
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user