mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df1b9efbbb | ||
|
|
553924fa1d | ||
|
|
a90dbd568f | ||
|
|
ca7ca1ce8e | ||
|
|
b3ab239a1e | ||
|
|
6e212c791c | ||
|
|
c6067ef147 | ||
|
|
8333f0c34e | ||
|
|
75becd8bb4 | ||
|
|
667376301a | ||
|
|
128e6b68a3 | ||
|
|
a7c2844ab4 | ||
|
|
bd0b7d274b | ||
|
|
bc57fad52b | ||
|
|
b81ef53cb9 | ||
|
|
c182bf0a08 | ||
|
|
07802b4b13 | ||
|
|
50c4d947ed | ||
|
|
b9b20e86dd | ||
|
|
64b93ce7e1 | ||
|
|
0bde4399e3 | ||
|
|
9b7818c5a7 | ||
|
|
1f2dcff353 | ||
|
|
4d82622fdb | ||
|
|
e7393caf01 | ||
|
|
8e5cb958ea | ||
|
|
bc2cf02ec6 | ||
|
|
7a8ecb306f | ||
|
|
76c3901afa | ||
|
|
f16b2227a1 | ||
|
|
d8d7ca5408 | ||
|
|
28c6b71e99 | ||
|
|
c40d3e2377 | ||
|
|
2217fea0ad | ||
|
|
94746441b5 | ||
|
|
c3e861fa8b | ||
|
|
ae1d644dba | ||
|
|
1fcb7e4e42 | ||
|
|
646cbba21c | ||
|
|
eef4d13235 | ||
|
|
9d1fe4c6b6 | ||
|
|
610d08f4ca | ||
|
|
7e52d0d6de | ||
|
|
4f0c5925ad |
@@ -2,4 +2,5 @@
|
||||
/**/*/node_modules
|
||||
/**/*/target
|
||||
/**/*/build-dir
|
||||
/**/*/java-runtime
|
||||
.idea
|
||||
@@ -1,5 +1,87 @@
|
||||
# Changelog
|
||||
|
||||
## [2.4.0] - 2019-11-23
|
||||
### Changed
|
||||
- Pimpandhost Host support
|
||||
|
||||
## [2.3.2] - 2019-11-23
|
||||
### Changed
|
||||
- Fix a bug with the scan component
|
||||
|
||||
## [2.3.1] - 2019-11-22
|
||||
### Changed
|
||||
- Fix connexion reset issue
|
||||
- Fix Post Status when autostart option is selected
|
||||
- Add stop all button
|
||||
|
||||
## [2.3.0] - 2019-11-21
|
||||
### Changed
|
||||
- UI enhancements and overhaul
|
||||
- Stability fixes with HTTP connections
|
||||
- MacOS official support
|
||||
- Use local sockjs and material icons font
|
||||
|
||||
## [2.2.5] - 2019-11-09
|
||||
### Changed
|
||||
- Exclude thumbs download for imagezilla and pixhost
|
||||
- Change retry on fail logic (may help avoid having download errors)
|
||||
|
||||
## [2.2.4] - 2019-10-05
|
||||
### Added
|
||||
- AppImage target for linux
|
||||
|
||||
## [2.2.3] - 2019-10-01
|
||||
### Changed
|
||||
- Strip libjvm.so for debian
|
||||
|
||||
## [2.2.2] - 2019-10-01
|
||||
### Changed
|
||||
- Update product name
|
||||
|
||||
## [2.2.1] - 2019-10-01
|
||||
### Changed
|
||||
- Update dependency list for deb package
|
||||
|
||||
## [2.2.0] - 2019-10-01
|
||||
### Added
|
||||
- Embedd JRE within the app
|
||||
### Changed
|
||||
- Fix 'Start' on pending items
|
||||
- Upgrade to Java 11
|
||||
|
||||
## [2.1.0] - 2019-09-30
|
||||
### Added
|
||||
- Add a create a subfolder per thread option
|
||||
|
||||
## [2.0.4] - 2019-09-27
|
||||
### Changed
|
||||
- Fix some issues with ImxHost
|
||||
|
||||
## [2.0.3] - 2019-09-23
|
||||
### Changed
|
||||
- Update deb dependencies
|
||||
|
||||
## [2.0.2] - 2019-09-23
|
||||
### Changed
|
||||
- Add portable mode
|
||||
|
||||
## [2.0.1] - 2019-09-22
|
||||
### Changed
|
||||
- Change app name
|
||||
|
||||
## [2.0.0] - 2019-09-22
|
||||
### Added
|
||||
- Add sub folder option
|
||||
- Add clear finished option
|
||||
### Changed
|
||||
- UI enhancements
|
||||
- Bug fixes
|
||||
- Change data and log paths
|
||||
|
||||
## [1.6.5] - 2019-09-18
|
||||
### Changed
|
||||
- electron update
|
||||
|
||||
## [1.6.4] - 2019-09-18
|
||||
### Changed
|
||||
- Change menu item name
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>1.6.4</version>
|
||||
<version>2.4.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
+28
-17
@@ -4,7 +4,6 @@ const url = require("url");
|
||||
const getPort = require("get-port");
|
||||
const { spawn } = require("child_process");
|
||||
const { ipcMain } = require("electron");
|
||||
const commandExists = require("command-exists").sync;
|
||||
const { dialog } = require("electron");
|
||||
const axios = require('axios');
|
||||
const appDir = process.env.APPDIR;
|
||||
@@ -23,27 +22,18 @@ process.on("uncaughtException", err => {
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
if (!commandExists("java")) {
|
||||
dialog.showErrorBox(
|
||||
"Java command is missing",
|
||||
"Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function createWindow() {
|
||||
let icon;
|
||||
if(process.platform === "win32") {
|
||||
icon = path.join(__dirname, `icon.ico`);
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
icon = path.join(__dirname, `icon.icns`);
|
||||
icon = __dirname + '/icon.png';
|
||||
}
|
||||
win = new BrowserWindow({
|
||||
width: 1024,
|
||||
height: 800,
|
||||
minWidth: 640,
|
||||
minHeight: 480,
|
||||
frame: false,
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
webPreferences: {
|
||||
nodeIntegration: true
|
||||
},
|
||||
@@ -51,6 +41,8 @@ function createWindow() {
|
||||
});
|
||||
|
||||
win.removeMenu();
|
||||
win.setMenu(null);
|
||||
win.setMenuBarVisibility(false);
|
||||
|
||||
win.loadURL(
|
||||
url.format({
|
||||
@@ -72,13 +64,32 @@ getPort().then(port => {
|
||||
ipcMain.on("get-port", event => {
|
||||
event.reply("port", port);
|
||||
});
|
||||
vripperServer = spawn("java", [
|
||||
let javaBinPath;
|
||||
if(appDir !== undefined) {
|
||||
javaBinPath = path.join(appDir, "java-runtime/bin/java");
|
||||
} else {
|
||||
if(process.platform === 'darwin') {
|
||||
javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
|
||||
} else {
|
||||
javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
|
||||
}
|
||||
}
|
||||
let jarPath;
|
||||
if(appDir !== undefined) {
|
||||
jarPath = path.join(appDir, "bin/vripper-server.jar");
|
||||
} else {
|
||||
if(process.platform === 'darwin') {
|
||||
jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
|
||||
} else {
|
||||
jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
|
||||
}
|
||||
}
|
||||
vripperServer = spawn(javaBinPath, [
|
||||
"-Xms256m",
|
||||
"-Xmx1024m",
|
||||
"-Dvripper.server.port=" + port,
|
||||
"-jar",
|
||||
appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
|
||||
path.join(app.getPath('exe'), "../bin/vripper-server.jar")
|
||||
jarPath
|
||||
], {
|
||||
stdio: 'ignore'
|
||||
});
|
||||
|
||||
Generated
+234
-421
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,14 @@
|
||||
{
|
||||
"name": "vripper-electron",
|
||||
"version": "1.6.4",
|
||||
"description": "",
|
||||
"version": "2.4.0",
|
||||
"description": "A ripper for vipergirls.to built using web technolgies",
|
||||
"main": "main.js",
|
||||
"author": "",
|
||||
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
|
||||
"homepage": "https://github.com/death-claw/vripper-project",
|
||||
"license": "ISC",
|
||||
"build": {
|
||||
"appId": "tn.mnlr.vripper",
|
||||
"productName": "vripper",
|
||||
"files": [
|
||||
"**/*",
|
||||
"!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
|
||||
@@ -29,29 +31,51 @@
|
||||
{
|
||||
"from": "../vripper-server/target/vripper-server-${version}-electron.jar",
|
||||
"to": "bin/vripper-server.jar"
|
||||
},
|
||||
{
|
||||
"from": "java-runtime",
|
||||
"to": "java-runtime"
|
||||
}
|
||||
],
|
||||
"win": {
|
||||
"target": "nsis"
|
||||
"icon": "icon.png",
|
||||
"target": [
|
||||
"nsis"
|
||||
]
|
||||
},
|
||||
"nsis": {
|
||||
"createDesktopShortcut": "always",
|
||||
"oneClick": false,
|
||||
"perMachine": false
|
||||
},
|
||||
"linux": {
|
||||
"synopsis": "vipergirls.to ripper",
|
||||
"category": "Utility",
|
||||
"packageCategory": "Utility",
|
||||
"icon": "icons",
|
||||
"target": [
|
||||
"zip"
|
||||
"AppImage"
|
||||
]
|
||||
},
|
||||
"mac": {
|
||||
"category": "public.app-category.utilities",
|
||||
"target": [
|
||||
"dmg"
|
||||
],
|
||||
"icon": "icon.png"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"start": "electron .",
|
||||
"dist": "node pre-build.js && build"
|
||||
"dist": "node pre-build.js && electron-builder"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^6.0.4",
|
||||
"electron-builder": "^20.44.4"
|
||||
"electron": "^6.0.9",
|
||||
"electron-builder": "^21.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^0.19.0",
|
||||
"cheerio": "^1.0.0-rc.3",
|
||||
"command-exists": "^1.2.8",
|
||||
"copy-dir": "^1.1.0",
|
||||
"custom-electron-titlebar": "^3.0.10",
|
||||
"electron-context-menu": "^0.13.0",
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>1.6.4</version>
|
||||
<version>2.4.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-electron</artifactId>
|
||||
<name>vripper-electron</name>
|
||||
|
||||
@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
|
||||
const fs = require('fs');
|
||||
const rimraf = require("rimraf");
|
||||
const copydir = require("copy-dir");
|
||||
|
||||
const { execSync } = require("child_process");
|
||||
|
||||
rimraf.sync('./build/vripper-ui');
|
||||
copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
|
||||
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
|
||||
|
||||
fs.writeFileSync('./build/vripper-ui/index.html', $.html());
|
||||
|
||||
console.log('Building runtime environment');
|
||||
rimraf.sync('java-runtime');
|
||||
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
|
||||
if(process.platform === 'linux') {
|
||||
console.log('Stripping libjvm.so');
|
||||
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
|
||||
}
|
||||
@@ -1,15 +1,3 @@
|
||||
console.log("renderer intialized");
|
||||
const customTitlebar = require("custom-electron-titlebar");
|
||||
const electron = require("electron").remote;
|
||||
const contextMenu = require("electron-context-menu");
|
||||
|
||||
contextMenu({});
|
||||
const menu = new electron.Menu();
|
||||
const titlebar = new customTitlebar.Titlebar({
|
||||
backgroundColor: customTitlebar.Color.fromHex("#000000"),
|
||||
titleHorizontalAlignment: 'left',
|
||||
icon: 'favicon.ico',
|
||||
shadow: true,
|
||||
menu: menu
|
||||
});
|
||||
titlebar.updateTitle("Viper Ripper");
|
||||
contextMenu({});
|
||||
@@ -5,14 +5,14 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>1.6.4</version>
|
||||
<version>2.4.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<name>vripper-server</name>
|
||||
<description>vripper-server</description>
|
||||
|
||||
<properties>
|
||||
<java.version>1.8</java.version>
|
||||
<java.version>11</java.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@@ -17,18 +17,11 @@ public class SpringContext implements ApplicationContextAware {
|
||||
|
||||
private static ConfigurableApplicationContext context;
|
||||
|
||||
/**
|
||||
* Returns the Spring managed bean instance of the given class type (if it exists).
|
||||
* Returns null otherwise.
|
||||
*
|
||||
* @param beanClass
|
||||
* @return
|
||||
*/
|
||||
public static <T extends Object> T getBean(Class<T> beanClass) {
|
||||
public static <T> T getBean(Class<T> beanClass) {
|
||||
return context.getBean(beanClass);
|
||||
}
|
||||
|
||||
public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
|
||||
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
|
||||
return context.getBeansOfType(beanClass);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package tn.mnlr.vripper;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
@@ -10,9 +12,9 @@ import org.springframework.stereotype.Component;
|
||||
import tn.mnlr.vripper.exception.VripperException;
|
||||
import tn.mnlr.vripper.services.AppSettingsService;
|
||||
import tn.mnlr.vripper.services.PersistenceService;
|
||||
import tn.mnlr.vripper.services.PostParser;
|
||||
import tn.mnlr.vripper.services.VipergirlsAuthService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.io.File;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
@@ -21,13 +23,9 @@ import java.util.concurrent.Executors;
|
||||
public class VripperApplication {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
|
||||
|
||||
public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json";
|
||||
|
||||
public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
try {
|
||||
SpringApplication.run(VripperApplication.class, args);
|
||||
} catch (Exception e) {
|
||||
@@ -47,6 +45,17 @@ public class VripperApplication {
|
||||
@Autowired
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
@Value("${base.dir}")
|
||||
private String baseDir;
|
||||
|
||||
@Getter
|
||||
private String dataPath;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
|
||||
@@ -65,23 +74,4 @@ public class VripperApplication {
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
|
||||
}
|
||||
}
|
||||
|
||||
@Component
|
||||
public class TestRunner implements CommandLineRunner {
|
||||
|
||||
@Autowired
|
||||
private PostParser postParser;
|
||||
|
||||
@Override
|
||||
public void run(String... args) throws Exception {
|
||||
|
||||
// long start = System.currentTimeMillis();
|
||||
// PostParser.VRThreadParser vrThreadParser = postParser.new VRThreadParser("600408");
|
||||
// vrThreadParser.getPostPublishProcessor().subscribe(e -> System.out.println("new Post: " + e.getPostId()));
|
||||
// vrThreadParser.parse();
|
||||
// System.out.println("Total time: " + (System.currentTimeMillis() - start));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ public class Image {
|
||||
this.postName = postName;
|
||||
this.host = host;
|
||||
this.index = index;
|
||||
status = Status.PENDING;
|
||||
status = Status.STOPPED;
|
||||
imageStateProcessor = BehaviorProcessor.create();
|
||||
}
|
||||
|
||||
@@ -80,7 +80,7 @@ public class Image {
|
||||
.subscribe();
|
||||
|
||||
current.set(0);
|
||||
status = Status.PENDING;
|
||||
status = Status.STOPPED;
|
||||
imageStateProcessor.onNext(this);
|
||||
}
|
||||
|
||||
|
||||
@@ -26,12 +26,12 @@ public class Post {
|
||||
|
||||
private String postId;
|
||||
|
||||
private String threadTitle;
|
||||
|
||||
private String threadId;
|
||||
|
||||
private String title;
|
||||
|
||||
private String postCounter;
|
||||
|
||||
private String url;
|
||||
|
||||
private List<Image> images;
|
||||
@@ -46,27 +46,20 @@ public class Post {
|
||||
|
||||
private boolean removed = false;
|
||||
|
||||
public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId) {
|
||||
private String forum;
|
||||
|
||||
public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) {
|
||||
this.title = title;
|
||||
this.url = url;
|
||||
this.images = images;
|
||||
this.metadata = metadata;
|
||||
this.postId = postId;
|
||||
this.forum = forum;
|
||||
this.threadId = threadId;
|
||||
if (this.images.contains(null)) {
|
||||
System.out.println("Oops");
|
||||
}
|
||||
this.images.stream().map(e -> {
|
||||
if (e == null) {
|
||||
System.out.println("Am the cause");
|
||||
}
|
||||
return e.getHost();
|
||||
}).collect(Collectors.toSet());
|
||||
this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
|
||||
this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
|
||||
this.threadTitle = threadTitle;
|
||||
this.hosts = this.images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet());
|
||||
total = images.size();
|
||||
status = Status.PENDING;
|
||||
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
|
||||
public void setAppStateService(AppStateService appStateService) {
|
||||
|
||||
+4
@@ -5,10 +5,14 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.processors.BehaviorProcessor;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public abstract class ImagePersistanceMixin {
|
||||
|
||||
@JsonSerialize(converter = HostToString.class)
|
||||
|
||||
+4
@@ -1,8 +1,12 @@
|
||||
package tn.mnlr.vripper.entities.mixin.persistance;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public abstract class PostPersistanceMixin {
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
@@ -3,10 +3,14 @@ package tn.mnlr.vripper.entities.mixin.ui;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.processors.BehaviorProcessor;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public abstract class ImageUIMixin {
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package tn.mnlr.vripper.entities.mixin.ui;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public abstract class PostUIMixin {
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DownloadException(Exception e) {
|
||||
public DownloadException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class HostException extends Exception {
|
||||
public HostException(Exception e) {
|
||||
public HostException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ public class HostException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public HostException(String message, Exception e) {
|
||||
public HostException(String message, Throwable e) {
|
||||
super(message,e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class HtmlProcessorException extends Exception {
|
||||
|
||||
public HtmlProcessorException(Exception e) {
|
||||
public HtmlProcessorException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public PostParseException(String message, Exception e) {
|
||||
public PostParseException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
|
||||
public PostParseException(Exception e) {
|
||||
public PostParseException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ public class VripperException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public VripperException(Exception e) {
|
||||
public VripperException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class XpathException extends Exception {
|
||||
public XpathException(Exception e) {
|
||||
public XpathException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
@@ -28,7 +29,7 @@ public class AcidimgHost extends Host {
|
||||
|
||||
private static final String host = "acidimg.cc";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
|
||||
public static final String IMG_XPATH = "//img[@class='centred']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -39,20 +40,25 @@ public class AcidimgHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = getResponse(url, context).getDocument();
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
logger.info(String.format("Click button found for %s", url));
|
||||
logger.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url);
|
||||
httpPost.addHeader("Referer", url);
|
||||
@@ -64,9 +70,9 @@ public class AcidimgHost extends Host {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
logger.info(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
|
||||
logger.info(String.format("Cleaning response for %s", httpPost));
|
||||
logger.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
logger.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
@@ -76,7 +82,7 @@ public class AcidimgHost extends Host {
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
@@ -87,7 +93,7 @@ public class AcidimgHost extends Host {
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ import org.apache.http.Header;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.impl.client.BasicCookieStore;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -25,9 +27,10 @@ import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Iterator;
|
||||
import java.util.Random;
|
||||
|
||||
@Service
|
||||
abstract public class Host {
|
||||
@@ -45,6 +48,9 @@ abstract public class Host {
|
||||
@Autowired
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
|
||||
@@ -56,57 +62,63 @@ abstract public class Host {
|
||||
|
||||
abstract public String getHost();
|
||||
|
||||
abstract public String getLookup();
|
||||
|
||||
public boolean isSupported(String url) {
|
||||
return url.contains(getHost());
|
||||
return url.contains(getLookup());
|
||||
}
|
||||
|
||||
public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
|
||||
|
||||
HttpClientContext context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
try {
|
||||
|
||||
appStateService.postDownloadingUpdate(image.getPostId());
|
||||
|
||||
imageFileData.setPageUrl(image.getUrl());
|
||||
|
||||
/**
|
||||
/*
|
||||
* HOST SPECIFIC
|
||||
*/
|
||||
logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
|
||||
setNameAndUrl(image.getUrl(), imageFileData);
|
||||
logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
|
||||
logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
|
||||
logger.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
|
||||
setNameAndUrl(image.getUrl(), imageFileData, context);
|
||||
logger.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
|
||||
logger.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
|
||||
|
||||
logger.info(String.format("Building image request for %s", image.getUrl()));
|
||||
logger.debug(String.format("Building image request for %s", image.getUrl()));
|
||||
setImageRequest(imageFileData);
|
||||
/**
|
||||
/*
|
||||
* END HOST SPECIFIC
|
||||
*/
|
||||
|
||||
String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
|
||||
logger.info(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
|
||||
logger.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
|
||||
imageFileData.setImageName(formatImageFileName);
|
||||
File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
|
||||
logger.info(String.format("Saving to %s", destinationFolder.getPath()));
|
||||
logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
|
||||
if (!destinationFolder.exists()) {
|
||||
logger.info(String.format("Creating %s", destinationFolder.getPath()));
|
||||
destinationFolder.mkdirs();
|
||||
logger.debug(String.format("Creating %s", destinationFolder.getPath()));
|
||||
if (destinationFolder.mkdirs()) {
|
||||
logger.debug(String.format("Folder %s is created", destinationFolder.toString()));
|
||||
}
|
||||
}
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
|
||||
logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
|
||||
logger.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest(), context)) {
|
||||
|
||||
if(response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
|
||||
File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
|
||||
InputStream downloadStream = response.getEntity().getContent();
|
||||
FileOutputStream fos = new FileOutputStream(outputFile);
|
||||
try {
|
||||
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
image.setTotal(response.getEntity().getContentLength());
|
||||
logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
|
||||
logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
|
||||
logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
|
||||
logger.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
|
||||
|
||||
byte[] buffer = new byte[READ_BUFFER_SIZE];
|
||||
int read;
|
||||
@@ -116,18 +128,11 @@ abstract public class Host {
|
||||
downloadSpeedService.increase(read);
|
||||
}
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} finally {
|
||||
if (downloadStream != null) {
|
||||
downloadStream.close();
|
||||
}
|
||||
if (fos != null) {
|
||||
fos.close();
|
||||
}
|
||||
}
|
||||
checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if(Thread.interrupted()) {
|
||||
if (Thread.interrupted()) {
|
||||
throw new InterruptedException("Download was interrupted");
|
||||
}
|
||||
throw new DownloadException(e);
|
||||
@@ -151,8 +156,8 @@ abstract public class Host {
|
||||
}
|
||||
try {
|
||||
File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
|
||||
if (outImage.exists()) {
|
||||
outImage.delete();
|
||||
if (outImage.exists() && outImage.delete()) {
|
||||
logger.debug(String.format("%s is deleted", outImage.toString()));
|
||||
}
|
||||
Files.move(outputFile.toPath(), outImage.toPath());
|
||||
} catch (Exception e) {
|
||||
@@ -160,32 +165,20 @@ abstract public class Host {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Just for testing, you may ignore
|
||||
* @throws Exception
|
||||
*/
|
||||
private void randomFail() throws Exception {
|
||||
Random random = new Random();
|
||||
int i = random.nextInt(100);
|
||||
if (i < 50) {
|
||||
throw new Exception("Purpose error");
|
||||
}
|
||||
}
|
||||
|
||||
protected final String getDefaultImageName(final String imgUrl) {
|
||||
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
|
||||
final String getDefaultImageName(final String imgUrl) {
|
||||
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
|
||||
logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
|
||||
return imgUrl;
|
||||
}
|
||||
|
||||
protected final Response getResponse(final String url) throws HostException {
|
||||
final Response getResponse(final String url, final HttpClientContext context) throws HostException {
|
||||
String basePage;
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url);
|
||||
Header[] headers;
|
||||
logger.info(String.format("Requesting %s", url));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
|
||||
logger.debug(String.format("Requesting %s", url));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
@@ -198,20 +191,34 @@ abstract public class Host {
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Cleaning %s response", url));
|
||||
logger.debug(String.format("Cleaning %s response", url));
|
||||
return new Response(htmlProcessorService.clean(basePage), headers);
|
||||
} catch (HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void setImageRequest(final ImageFileData imageFileData) {
|
||||
protected String appendUri(String uri, String appendQuery) throws URISyntaxException {
|
||||
URI oldUri = new URI(uri);
|
||||
|
||||
String newQuery = oldUri.getQuery();
|
||||
if (newQuery == null) {
|
||||
newQuery = appendQuery;
|
||||
} else {
|
||||
newQuery += "&" + appendQuery;
|
||||
}
|
||||
|
||||
return new URI(oldUri.getScheme(), oldUri.getAuthority(),
|
||||
oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
|
||||
}
|
||||
|
||||
private void setImageRequest(final ImageFileData imageFileData) {
|
||||
HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
|
||||
httpGet.addHeader("Referer", imageFileData.getPageUrl());
|
||||
imageFileData.setImageRequest(httpGet);
|
||||
}
|
||||
|
||||
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
|
||||
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
|
||||
|
||||
@Getter
|
||||
public static class Response {
|
||||
@@ -219,6 +226,7 @@ abstract public class Host {
|
||||
this.document = document;
|
||||
this.headers = headers;
|
||||
}
|
||||
|
||||
private Document document;
|
||||
private Header[] headers;
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -18,8 +18,6 @@ import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class ImageBamHost extends Host {
|
||||
@@ -27,8 +25,8 @@ public class ImageBamHost extends Host {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
|
||||
|
||||
private static final String host = "imagebam.com";
|
||||
public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
|
||||
public static final String IMG_XPATH = "//img[@class='image']";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
|
||||
private static final String IMG_XPATH = "//img[@class='image']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -39,34 +37,31 @@ public class ImageBamHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Response response = getResponse(url);
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Response response = getResponse(url, context);
|
||||
Document doc = response.getDocument();
|
||||
Header[] headers = response.getHeaders();
|
||||
String cookies = Arrays.asList(headers)
|
||||
.stream()
|
||||
.filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
|
||||
.map(e -> e.getValue().split(";")[0].trim())
|
||||
.collect(Collectors.joining("; "));
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
logger.info(String.format("Getting cookies to use for %s", url));
|
||||
logger.info(String.format("Click button found for %s", url));
|
||||
logger.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url);
|
||||
httpGet.addHeader("Referer", url);
|
||||
httpGet.addHeader("Cookie", cookies);
|
||||
logger.info(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
|
||||
logger.debug(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
String s = EntityUtils.toString(res.getEntity());
|
||||
logger.debug(String.format("%s response is:%n%s", httpGet, s));
|
||||
logger.debug(String.format("Cleaning response for %s", httpGet));
|
||||
@@ -79,14 +74,14 @@ public class ImageBamHost extends Host {
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -16,7 +17,7 @@ import java.util.Optional;
|
||||
@Service
|
||||
public class ImageTwistHost extends Host {
|
||||
|
||||
public static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
|
||||
private static final String host = "imagetwist.com";
|
||||
@Autowired
|
||||
@@ -28,20 +29,25 @@ public class ImageTwistHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -17,7 +18,8 @@ public class ImageZillaHost extends Host {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
|
||||
|
||||
private static final String host = "imagezilla.net";
|
||||
public static final String IMG_XPATH = "//img[@id='photo']";
|
||||
private static final String lookup = "imagezilla.net/show";
|
||||
private static final String IMG_XPATH = "//img[@id='photo']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -28,15 +30,20 @@ public class ImageZillaHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
|
||||
logger.info(String.format("Resolving name for %s", url));
|
||||
logger.debug(String.format("Resolving name for %s", url));
|
||||
if(titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -17,7 +18,7 @@ public class ImgboxHost extends Host {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
|
||||
|
||||
private static final String host = "imgbox.com";
|
||||
public static final String IMG_XPATH = "//img[@id='img']";
|
||||
private static final String IMG_XPATH = "//img[@id='img']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -28,20 +29,25 @@ public class ImgboxHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
@@ -29,8 +30,8 @@ public class ImxHost extends Host {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
|
||||
|
||||
private static final String host = "imx.to";
|
||||
public static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
|
||||
public static final String IMG_XPATH = "//img[@class='centred']";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -41,50 +42,62 @@ public class ImxHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Response resp = getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node contDiv;
|
||||
String value = null;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
Node node = contDiv.getAttributes().getNamedItem("value");
|
||||
if (node != null) {
|
||||
value = node.getTextContent();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
logger.info(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url);
|
||||
httpPost.addHeader("Referer", url);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", "Continue to image ... "));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
logger.info(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
|
||||
logger.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
if (value == null) {
|
||||
throw new HostException("Failed to obtain value attribute from continue input");
|
||||
}
|
||||
logger.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", value));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
logger.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
logger.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
|
||||
@Service
|
||||
public class PimpandhostHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PimpandhostHost.class);
|
||||
|
||||
private static final String host = "pimpandhost.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url;
|
||||
try {
|
||||
url = appendUri(_url.replace("http://", "https://"), "size=original");
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Response resp = getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -17,7 +18,8 @@ public class PixhostHost extends Host {
|
||||
private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
|
||||
|
||||
private static final String host = "pixhost.to";
|
||||
public static final String IMG_XPATH = "//img[@id='image']";
|
||||
private static final String lookup = "pixhost.to/show";
|
||||
private static final String IMG_XPATH = "//img[@id='image']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -28,20 +30,25 @@ public class PixhostHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
logger.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -17,8 +18,8 @@ public class TurboImageHost extends Host {
|
||||
private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
|
||||
|
||||
private static final String host = "turboimagehost.com";
|
||||
public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
|
||||
public static final String IMG_XPATH = "//img[@id='uImage']";
|
||||
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
|
||||
private static final String IMG_XPATH = "//img[@id='uImage']";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
@@ -29,15 +30,20 @@ public class TurboImageHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
logger.info(String.format("Resolving name for %s", url));
|
||||
logger.debug(String.format("Resolving name for %s", url));
|
||||
if(titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
|
||||
@@ -18,14 +18,14 @@ public class DownloadJob implements Callable<Image> {
|
||||
@Getter
|
||||
private final ImageFileData imageFileData = new ImageFileData();
|
||||
|
||||
public DownloadJob(Image image) {
|
||||
DownloadJob(Image image) {
|
||||
this.image = image;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Image call() throws Exception {
|
||||
|
||||
logger.info(String.format("Starting downloading %s", image.getUrl()));
|
||||
logger.debug(String.format("Starting downloading %s", image.getUrl()));
|
||||
image.setStatus(Image.Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
image.getHost().download(image, imageFileData);
|
||||
|
||||
@@ -36,7 +36,7 @@ public class DownloadQ {
|
||||
|
||||
public void put(Image image) throws InterruptedException {
|
||||
synchronized (appStateService) {
|
||||
logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
DownloadJob downloadJob = new DownloadJob(image);
|
||||
downloadQ.put(downloadJob);
|
||||
@@ -44,9 +44,9 @@ public class DownloadQ {
|
||||
}
|
||||
}
|
||||
|
||||
public DownloadJob take() throws InterruptedException {
|
||||
DownloadJob take() throws InterruptedException {
|
||||
DownloadJob downloadJob = downloadQ.take();
|
||||
logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
|
||||
logger.debug(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
|
||||
return downloadJob;
|
||||
}
|
||||
|
||||
@@ -71,17 +71,17 @@ public class DownloadQ {
|
||||
return;
|
||||
}
|
||||
appStateService.getPost(postId).setStatus(Post.Status.PENDING);
|
||||
logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
for (Image image : images) {
|
||||
put(image);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void removeScheduled(Image image) {
|
||||
private void removeScheduled(Image image) {
|
||||
synchronized (appStateService) {
|
||||
image.setStatus(Image.Status.STOPPED);
|
||||
logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
|
||||
logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
|
||||
|
||||
Iterator<DownloadJob> iterator = downloadQ.iterator();
|
||||
boolean removed = false;
|
||||
@@ -90,22 +90,22 @@ public class DownloadQ {
|
||||
if (next.getImage().getPostId().equals(image.getPostId())) {
|
||||
iterator.remove();
|
||||
appStateService.doneDownloadJob(image);
|
||||
logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
|
||||
logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
|
||||
removed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!removed) {
|
||||
logger.warn(String.format("Job for %s does not exist", image.getUrl()));
|
||||
logger.debug(String.format("Job for %s does not exist", image.getUrl()));
|
||||
}
|
||||
|
||||
image.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
public void removeRunning(String postId) {
|
||||
logger.info(String.format("Interrupting running jobs for post id %s", postId));
|
||||
private void removeRunning(String postId) {
|
||||
logger.debug(String.format("Interrupting running jobs for post id %s", postId));
|
||||
executionService.stop(postId);
|
||||
}
|
||||
|
||||
@@ -126,8 +126,8 @@ public class DownloadQ {
|
||||
if (images.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
|
||||
images.forEach(image -> removeScheduled(image));
|
||||
logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
|
||||
images.forEach(this::removeScheduled);
|
||||
removeRunning(postId);
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -13,7 +13,7 @@ import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.time.Duration;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
@@ -36,7 +36,7 @@ public class ExecutionService {
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
|
||||
private AtomicInteger threadCount = new AtomicInteger();
|
||||
private final AtomicInteger threadCount = new AtomicInteger();
|
||||
|
||||
private ExecutorService executor = Executors.newFixedThreadPool(10);
|
||||
|
||||
@@ -53,8 +53,8 @@ public class ExecutionService {
|
||||
|
||||
retryPolicy = new RetryPolicy<>()
|
||||
.handleIf(e -> !(e instanceof InterruptedException))
|
||||
.withDelay(Duration.ofSeconds(5))
|
||||
.withMaxRetries(2)
|
||||
.withBackoff(10, 60, ChronoUnit.SECONDS)
|
||||
.withMaxRetries(4)
|
||||
.abortOn(InterruptedException.class)
|
||||
.onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
|
||||
|
||||
@@ -68,7 +68,7 @@ public class ExecutionService {
|
||||
executionThread.interrupt();
|
||||
executor.shutdown();
|
||||
appStateService.getCurrentPosts().keySet().forEach(p -> {
|
||||
logger.info(String.format("Stopping download jobs for %s", p));
|
||||
logger.debug(String.format("Stopping download jobs for %s", p));
|
||||
this.stop(p);
|
||||
});
|
||||
executor.awaitTermination(10, TimeUnit.SECONDS);
|
||||
@@ -80,7 +80,7 @@ public class ExecutionService {
|
||||
.filter(e -> e.getImage().getPostId().equals(postId))
|
||||
.peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
|
||||
.collect(Collectors.toList());
|
||||
logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
|
||||
logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
|
||||
|
||||
data.forEach(e -> {
|
||||
futures.get(e.getImage().getUrl()).cancel(true);
|
||||
@@ -91,7 +91,7 @@ public class ExecutionService {
|
||||
});
|
||||
}
|
||||
|
||||
boolean canRun() {
|
||||
private boolean canRun() {
|
||||
boolean canRun = threadCount.get() < settings.getMaxThreads();
|
||||
if (canRun && downloadQ.isNotPauseQ()) {
|
||||
threadCount.incrementAndGet();
|
||||
@@ -103,7 +103,7 @@ public class ExecutionService {
|
||||
public void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
if (canRun()) {
|
||||
DownloadJob take = null;
|
||||
DownloadJob take;
|
||||
try {
|
||||
take = downloadQ.take();
|
||||
if (take == null) {
|
||||
@@ -121,7 +121,7 @@ public class ExecutionService {
|
||||
Failsafe.with(retryPolicy)
|
||||
.onFailure(e -> {
|
||||
if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
|
||||
logger.info("Job successfully interrupted");
|
||||
logger.debug("Job successfully interrupted");
|
||||
return;
|
||||
}
|
||||
logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
|
||||
@@ -129,7 +129,7 @@ public class ExecutionService {
|
||||
})
|
||||
.onComplete(e -> {
|
||||
appStateService.doneDownloadJob(finalTake.getImage());
|
||||
logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
|
||||
logger.debug(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
|
||||
synchronized (threadCount) {
|
||||
threadCount.decrementAndGet();
|
||||
running.remove(finalTake);
|
||||
@@ -139,7 +139,7 @@ public class ExecutionService {
|
||||
})
|
||||
.get(finalTake::call);
|
||||
};
|
||||
logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
|
||||
logger.debug(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
|
||||
futures.put(finalTake.getImage().getUrl(), executor.submit(task));
|
||||
} else {
|
||||
synchronized (threadCount) {
|
||||
|
||||
@@ -36,6 +36,9 @@ public class AppSettingsService {
|
||||
private final String V_THANKS = "VTHANKS";
|
||||
private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
|
||||
private final String FORCE_ORDER = "FORCE_ORDER";
|
||||
private final String SUBFOLDER = "SUBFOLDER";
|
||||
private final String THREADSUBFOLDER = "THREADSUBFOLDER";
|
||||
private final String CLEAR = "CLEAR";
|
||||
private final String DARK_THEME = "DARK_THEME";
|
||||
|
||||
private String downloadPath;
|
||||
@@ -46,7 +49,10 @@ public class AppSettingsService {
|
||||
private String vPassword;
|
||||
private boolean vThanks;
|
||||
private boolean desktopClipboard;
|
||||
private boolean subLocation;
|
||||
private boolean threadSubLocation;
|
||||
private boolean forceOrder;
|
||||
private boolean clearCompleted;
|
||||
private boolean darkTheme;
|
||||
|
||||
public void setVPassword(String vPassword) {
|
||||
@@ -68,6 +74,9 @@ public class AppSettingsService {
|
||||
vThanks = prefs.getBoolean(V_THANKS, false);
|
||||
desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
|
||||
forceOrder = prefs.getBoolean(FORCE_ORDER, false);
|
||||
subLocation = prefs.getBoolean(SUBFOLDER, false);
|
||||
threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
|
||||
clearCompleted = prefs.getBoolean(CLEAR, false);
|
||||
darkTheme = prefs.getBoolean(DARK_THEME, false);
|
||||
}
|
||||
|
||||
@@ -83,6 +92,9 @@ public class AppSettingsService {
|
||||
prefs.putBoolean(V_THANKS, vThanks);
|
||||
prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
|
||||
prefs.putBoolean(FORCE_ORDER, forceOrder);
|
||||
prefs.putBoolean(SUBFOLDER, subLocation);
|
||||
prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
|
||||
prefs.putBoolean(CLEAR, clearCompleted);
|
||||
prefs.putBoolean(DARK_THEME, darkTheme);
|
||||
|
||||
try {
|
||||
@@ -106,8 +118,8 @@ public class AppSettingsService {
|
||||
throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
|
||||
}
|
||||
|
||||
if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 8) {
|
||||
throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 8));
|
||||
if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
|
||||
throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,7 +139,7 @@ public class AppSettingsService {
|
||||
@JsonProperty("darkTheme")
|
||||
private boolean darkTheme;
|
||||
|
||||
public Theme(boolean darkTheme) {
|
||||
Theme(boolean darkTheme) {
|
||||
this.darkTheme = darkTheme;
|
||||
}
|
||||
}
|
||||
@@ -153,8 +165,14 @@ public class AppSettingsService {
|
||||
private boolean desktopClipboard;
|
||||
@JsonProperty("forceOrder")
|
||||
private boolean forceOrder;
|
||||
@JsonProperty("subLocation")
|
||||
private boolean subLocation;
|
||||
@JsonProperty("threadSubLocation")
|
||||
private boolean threadSubLocation;
|
||||
@JsonProperty("clearCompleted")
|
||||
private boolean clearCompleted;
|
||||
|
||||
public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder) {
|
||||
public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted) {
|
||||
this.downloadPath = downloadPath;
|
||||
this.maxThreads = maxThreads;
|
||||
this.autoStart = autoStart;
|
||||
@@ -164,6 +182,9 @@ public class AppSettingsService {
|
||||
this.vThanks = vThanks;
|
||||
this.desktopClipboard = desktopClipboard;
|
||||
this.forceOrder = forceOrder;
|
||||
this.subLocation = subLocation;
|
||||
this.threadSubLocation = threadSubLocation;
|
||||
this.clearCompleted = clearCompleted;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
import tn.mnlr.vripper.q.DownloadJob;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@@ -37,6 +36,9 @@ public class AppStateService {
|
||||
@Autowired
|
||||
private PersistenceService persistenceService;
|
||||
|
||||
@Autowired
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
public void onImageUpdate(Image imageState) {
|
||||
|
||||
persistenceService.getProcessor().onNext(currentPosts);
|
||||
@@ -56,9 +58,12 @@ public class AppStateService {
|
||||
public synchronized void newDownloadJob(DownloadJob downloadJob) {
|
||||
String postId = downloadJob.getImage().getPostId();
|
||||
checkKeyRunningPosts(postId);
|
||||
int i = runningPosts.get(postId).incrementAndGet();
|
||||
if (i > 0) {
|
||||
Post post = currentPosts.get(postId);
|
||||
runningPosts.get(postId).incrementAndGet();
|
||||
}
|
||||
|
||||
public synchronized void postDownloadingUpdate(String postId) {
|
||||
Post post = currentPosts.get(postId);
|
||||
if (!post.getStatus().equals(Post.Status.DOWNLOADING)) {
|
||||
post.setStatus(Post.Status.DOWNLOADING);
|
||||
livePostsState.onNext(post);
|
||||
}
|
||||
@@ -72,14 +77,16 @@ public class AppStateService {
|
||||
post.setStatus(Post.Status.PARTIAL);
|
||||
}
|
||||
if (i == 0) {
|
||||
if (post.getImages().stream().map(Image::getStatus).filter(e -> e.equals(Image.Status.ERROR)).count() > 0) {
|
||||
if (post.getImages().stream().map(Image::getStatus).anyMatch(e -> e.equals(Image.Status.ERROR))) {
|
||||
post.setStatus(Post.Status.ERROR);
|
||||
} else {
|
||||
if (!Post.Status.STOPPED.equals(post.getStatus())) {
|
||||
post.setStatus(Post.Status.COMPLETE);
|
||||
if (appSettingsService.isClearCompleted()) {
|
||||
remove(image.getPostId());
|
||||
}
|
||||
}
|
||||
}
|
||||
livePostsState.onNext(post);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,13 +100,7 @@ public class AppStateService {
|
||||
currentPosts.get(postId).setRemoved(true);
|
||||
runningPosts.remove(postId);
|
||||
currentPosts.remove(postId);
|
||||
Iterator<Map.Entry<String, Image>> iterator = currentImages.entrySet().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Map.Entry<String, Image> entry = iterator.next();
|
||||
if(entry.getValue().getPostId().equals(postId)) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
currentImages.entrySet().removeIf(entry -> entry.getValue().getPostId().equals(postId));
|
||||
persistenceService.getProcessor().onNext(currentPosts);
|
||||
}
|
||||
|
||||
@@ -124,6 +125,7 @@ public class AppStateService {
|
||||
return toRemove;
|
||||
}
|
||||
|
||||
|
||||
@Getter
|
||||
public static class CachedThread {
|
||||
|
||||
|
||||
@@ -4,15 +4,19 @@ import org.apache.http.client.config.CookieSpecs;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.client.LaxRedirectStrategy;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
public class ConnectionManager {
|
||||
|
||||
private ConnectionManager() {
|
||||
@@ -31,14 +35,21 @@ public class ConnectionManager {
|
||||
private void buildConnectionPool() {
|
||||
|
||||
pcm = new PoolingHttpClientConnectionManager();
|
||||
pcm.setMaxTotal(200);
|
||||
pcm.setDefaultMaxPerRoute(10);
|
||||
pcm.setMaxTotal(50);
|
||||
pcm.setDefaultMaxPerRoute(4);
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 1000)
|
||||
private void idleConnectionMonitoring() {
|
||||
pcm.closeExpiredConnections();
|
||||
pcm.closeIdleConnections(30, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
public HttpClientBuilder getClient() {
|
||||
return HttpClients.custom()
|
||||
.setConnectionManager(pcm)
|
||||
.setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
|
||||
.setRedirectStrategy(new LaxRedirectStrategy())
|
||||
.disableAutomaticRetries()
|
||||
.setDefaultRequestConfig(rc);
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ public class GlobalState {
|
||||
private long remaining;
|
||||
private long error;
|
||||
|
||||
public GlobalState(long running, long queued, long remaining, long error) {
|
||||
GlobalState(long running, long queued, long remaining, long error) {
|
||||
this.running = running;
|
||||
this.queued = queued;
|
||||
this.remaining = remaining;
|
||||
@@ -25,11 +25,7 @@ public class GlobalState {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
GlobalState that = (GlobalState) o;
|
||||
return running == that.running &&
|
||||
queued == that.queued &&
|
||||
remaining == that.remaining &&
|
||||
error == that.error &&
|
||||
Objects.equals(type, that.type);
|
||||
return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -29,6 +29,9 @@ public class GlobalStateService {
|
||||
@Getter
|
||||
private PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
|
||||
|
||||
public GlobalStateService() {
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 3000)
|
||||
private void interval() {
|
||||
GlobalState newGlobalState = new GlobalState(
|
||||
|
||||
@@ -11,10 +11,6 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
@Service
|
||||
public class HtmlProcessorService {
|
||||
|
||||
private HtmlProcessorService() {
|
||||
|
||||
}
|
||||
|
||||
public Document clean(String htmlContent) throws HtmlProcessorException {
|
||||
try {
|
||||
TagNode clean = new HtmlCleaner().clean(htmlContent);
|
||||
|
||||
@@ -4,6 +4,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
@@ -19,8 +20,12 @@ public class PathService {
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
public final File getDownloadDestinationFolder(String postId) {
|
||||
String postTitle = appStateService.getCurrentPosts().get(postId).getTitle();
|
||||
return new File(appSettingsService.getDownloadPath(), sanitize(postTitle + "_" + postId));
|
||||
Post post = appStateService.getCurrentPosts().get(postId);
|
||||
String postTitle = post.getTitle();
|
||||
String threadTitle = post.getThreadTitle();
|
||||
File sourceFolder = appSettingsService.isSubLocation() ? new File(appSettingsService.getDownloadPath(), sanitize(post.getForum())) : new File(appSettingsService.getDownloadPath());
|
||||
sourceFolder = appSettingsService.isThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
|
||||
return new File(sourceFolder, sanitize(postTitle + "_" + postId));
|
||||
}
|
||||
|
||||
public final String sanitize(final String folderName) {
|
||||
@@ -32,8 +37,8 @@ public class PathService {
|
||||
/**
|
||||
* Will sanitize the image name and remove extension
|
||||
*
|
||||
* @param imageName
|
||||
* @return
|
||||
* @param imageName path string
|
||||
* @return Sanitized local path string
|
||||
*/
|
||||
public final String formatImageFileName(String imageName) {
|
||||
int extensionIndex = imageName.lastIndexOf('.');
|
||||
|
||||
@@ -8,12 +8,14 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.VripperApplication;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
|
||||
import tn.mnlr.vripper.entities.mixin.persistance.PostPersistanceMixin;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
@@ -23,10 +25,10 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class PersistenceService {
|
||||
@@ -36,6 +38,9 @@ public class PersistenceService {
|
||||
@Autowired
|
||||
private AppStateService stateService;
|
||||
|
||||
@Autowired
|
||||
private VripperApplication.AppCommandRunner appCommandRunner;
|
||||
|
||||
private ObjectMapper om;
|
||||
|
||||
private Disposable subscription;
|
||||
@@ -44,22 +49,6 @@ public class PersistenceService {
|
||||
private PublishProcessor<Map<String, Post>> processor = PublishProcessor.create();
|
||||
|
||||
private PersistenceService() {
|
||||
File dataFile = new File(VripperApplication.dataPath);
|
||||
if(!dataFile.exists()) {
|
||||
try {
|
||||
if(dataFile.createNewFile()) {
|
||||
logger.info("Data file successfully created");
|
||||
try (FileWriter fw = new FileWriter(dataFile)) {
|
||||
fw.write("{}");
|
||||
}
|
||||
} else {
|
||||
logger.info("Data file already exists");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("Unable to create data file", e);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
om = new ObjectMapper();
|
||||
om.addMixIn(Image.class, ImagePersistanceMixin.class);
|
||||
om.addMixIn(Post.class, PostPersistanceMixin.class);
|
||||
@@ -72,6 +61,34 @@ public class PersistenceService {
|
||||
.subscribe();
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
File dataFile = new File(appCommandRunner.getDataPath());
|
||||
if (!dataFile.exists()) {
|
||||
try {
|
||||
if (dataFile.getParentFile().mkdirs()) {
|
||||
logger.debug(String.format("%s is created", dataFile.getParentFile().toString()));
|
||||
}
|
||||
if (!dataFile.getParentFile().isDirectory() || !dataFile.getParentFile().canWrite()) {
|
||||
logger.error(String.format("Unable to write in %s", dataFile.getParent()));
|
||||
SpringContext.close();
|
||||
}
|
||||
|
||||
if (dataFile.createNewFile()) {
|
||||
logger.debug("Data file successfully created");
|
||||
try (FileWriter fw = new FileWriter(dataFile)) {
|
||||
fw.write("{}");
|
||||
}
|
||||
} else {
|
||||
logger.warn("Data file already exists");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("Unable to create data file", e);
|
||||
SpringContext.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void preDestroy() {
|
||||
this.subscription.dispose();
|
||||
@@ -80,9 +97,9 @@ public class PersistenceService {
|
||||
this.persist(stateService.getCurrentPosts());
|
||||
}
|
||||
|
||||
public void persist(Map<String, Post> currentPosts) {
|
||||
private void persist(Map<String, Post> currentPosts) {
|
||||
|
||||
try (PrintWriter out = new PrintWriter(VripperApplication.dataPath, "UTF-8")) {
|
||||
try (PrintWriter out = new PrintWriter(appCommandRunner.getDataPath(), StandardCharsets.UTF_8)) {
|
||||
out.print(om.writeValueAsString(currentPosts));
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to persist app state", e);
|
||||
@@ -95,6 +112,14 @@ public class PersistenceService {
|
||||
return om.readValue(content, om.getTypeFactory().constructMapType(HashMap.class, String.class, Post.class));
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to read app state", e);
|
||||
long timestamp = new Date().getTime();
|
||||
logger.warn(String.format("trying to rename old data file from %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + "." + timestamp + ".old"));
|
||||
try {
|
||||
Files.move(new File(appCommandRunner.getDataPath()).toPath(), new File(appCommandRunner.getDataPath() + "." + timestamp + ".old").toPath());
|
||||
} catch (IOException ex) {
|
||||
logger.error(String.format("Failed to rename %s to %s", appCommandRunner.getDataPath(), appCommandRunner.getDataPath() + ".old"));
|
||||
SpringContext.close();
|
||||
}
|
||||
}
|
||||
|
||||
return new HashMap<>();
|
||||
@@ -102,12 +127,12 @@ public class PersistenceService {
|
||||
|
||||
public void restore() {
|
||||
|
||||
String jsonContent;
|
||||
String jsonContent = null;
|
||||
try {
|
||||
jsonContent = Files.readAllLines(Paths.get(VripperApplication.dataPath), StandardCharsets.UTF_8).stream().collect(Collectors.joining());
|
||||
jsonContent = String.join("", Files.readAllLines(Paths.get(appCommandRunner.getDataPath()), StandardCharsets.UTF_8));
|
||||
} catch (Exception e) {
|
||||
logger.warn("data file not found, previous state cannot be restored", e);
|
||||
return;
|
||||
logger.error("data file cannot be read, previous state cannot be restored", e);
|
||||
SpringContext.close();
|
||||
}
|
||||
|
||||
Map<String, Post> read = read(jsonContent);
|
||||
@@ -115,7 +140,7 @@ public class PersistenceService {
|
||||
stateService.getCurrentPosts().clear();
|
||||
stateService.getCurrentPosts().putAll(read);
|
||||
stateService.getCurrentPosts().values().forEach(p -> {
|
||||
if(Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL).contains(p.getStatus())) {
|
||||
if (Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL, Post.Status.PENDING).contains(p.getStatus())) {
|
||||
p.setStatus(Post.Status.STOPPED);
|
||||
}
|
||||
});
|
||||
@@ -126,8 +151,6 @@ public class PersistenceService {
|
||||
stateService.getCurrentImages().put(e.getUrl(), e);
|
||||
});
|
||||
|
||||
read.values().forEach(e -> {
|
||||
e.setAppStateService(stateService);
|
||||
});
|
||||
read.values().forEach(e -> e.setAppStateService(stateService));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +1,27 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import io.reactivex.processors.PublishProcessor;
|
||||
import lombok.Getter;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.utils.URIBuilder;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.xml.sax.Attributes;
|
||||
import org.xml.sax.SAXException;
|
||||
import org.xml.sax.helpers.DefaultHandler;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.q.DownloadQ;
|
||||
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class PostParser {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PostParser.class);
|
||||
|
||||
private static final String VR_API = "https://vipergirls.to/vr.php";
|
||||
static final String VR_API = "https://vipergirls.to/vr.php";
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
|
||||
SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||
|
||||
@Autowired
|
||||
private List<Host> supportedHosts;
|
||||
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
|
||||
@@ -60,25 +37,31 @@ public class PostParser {
|
||||
@Autowired
|
||||
private VipergirlsAuthService vipergirlsAuthService;
|
||||
|
||||
public PostParser() throws ParserConfigurationException, SAXException {
|
||||
@Autowired
|
||||
private List<Host> supportedHosts;
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
|
||||
}
|
||||
|
||||
public void addPost(String postId, String threadId) throws PostParseException {
|
||||
|
||||
if (appStateService.getCurrentPosts().containsKey(postId)) {
|
||||
logger.info(String.format("skipping %s, already loaded", postId));
|
||||
logger.warn(String.format("skipping %s, already loaded", postId));
|
||||
return;
|
||||
}
|
||||
|
||||
VRPostParser vrPostParser = new VRPostParser(threadId, postId);
|
||||
VRPostParser vrPostParser = new VRPostParser(threadId, postId, cm, vipergirlsAuthService, supportedHosts);
|
||||
Post post = vrPostParser.parse();
|
||||
|
||||
post.setAppStateService(appStateService);
|
||||
post.getImages().forEach(e -> e.setAppStateService(appStateService));
|
||||
authService.leaveThanks(post.getUrl(), post.getPostId());
|
||||
if (appSettingsService.isAutoStart()) {
|
||||
logger.info("Auto start downloads option is enabled");
|
||||
logger.info(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
|
||||
logger.debug("Auto start downloads option is enabled");
|
||||
logger.debug(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
|
||||
post.setStatus(Post.Status.PENDING);
|
||||
try {
|
||||
downloadQ.enqueue(post);
|
||||
} catch (InterruptedException e) {
|
||||
@@ -86,300 +69,14 @@ public class PostParser {
|
||||
Thread.currentThread().interrupt();
|
||||
return;
|
||||
}
|
||||
logger.info(String.format("Done enqueuing jobs for %s", post.getUrl()));
|
||||
logger.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
|
||||
} else {
|
||||
logger.info("Auto start downloads option is disabled");
|
||||
post.setStatus(Post.Status.STOPPED);
|
||||
logger.debug("Auto start downloads option is disabled");
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static abstract class VRPostState {
|
||||
private final String threadId;
|
||||
|
||||
protected VRPostState(String threadId) {
|
||||
this.threadId = threadId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class VRPostParse extends VRPostState {
|
||||
|
||||
private final String type = "postParse";
|
||||
private String postId;
|
||||
private int number;
|
||||
private String title;
|
||||
private int imageCount;
|
||||
private String url;
|
||||
private List<String> previews;
|
||||
|
||||
|
||||
public VRPostParse(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews) {
|
||||
super(threadId);
|
||||
this.postId = postId;
|
||||
this.number = number;
|
||||
this.title = title;
|
||||
this.imageCount = imageCount;
|
||||
this.previews = previews;
|
||||
this.url = url;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class VRThreadParseState extends VRPostState {
|
||||
|
||||
private final String type = "threadParseState";
|
||||
private final String state;
|
||||
|
||||
public VRThreadParseState(String threadId, String state) {
|
||||
super(threadId);
|
||||
this.state = state;
|
||||
}
|
||||
}
|
||||
|
||||
public class VRThreadParser {
|
||||
|
||||
@Getter
|
||||
private final PublishProcessor<VRPostState> postPublishProcessor = PublishProcessor.create();
|
||||
|
||||
private String threadId;
|
||||
|
||||
public VRThreadParser(String threadId) {
|
||||
this.threadId = threadId;
|
||||
}
|
||||
|
||||
public Void parse() throws PostParseException {
|
||||
|
||||
logger.info(String.format("Parsing thread %s", threadId));
|
||||
HttpGet httpGet;
|
||||
try {
|
||||
URIBuilder uriBuilder = new URIBuilder(VR_API);
|
||||
uriBuilder.setParameter("t", threadId);
|
||||
httpGet = cm.buildHttpGet(uriBuilder.build());
|
||||
if (vipergirlsAuthService.getCookies() != null) {
|
||||
httpGet.addHeader("Cookie", vipergirlsAuthService.getCookies());
|
||||
}
|
||||
} catch (URISyntaxException e) {
|
||||
throw new PostParseException(e);
|
||||
}
|
||||
VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
|
||||
HttpClient connection = cm.getClient().build();
|
||||
logger.info(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
|
||||
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
logger.error("parsing failed", e);
|
||||
throw new PostParseException(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private class VRThreadHandler extends DefaultHandler {
|
||||
|
||||
private final String threadId;
|
||||
private final PublishProcessor<VRPostState> vrPostPublishProcessor;
|
||||
private String threadTitle;
|
||||
private String postId;
|
||||
private String postTitle;
|
||||
private int imageCount;
|
||||
private int postCounter;
|
||||
private int previewCounter = 0;
|
||||
private List<String> previews = new ArrayList<>();
|
||||
|
||||
public VRThreadHandler(String threadId, PublishProcessor<VRPostState> vrPostPublishProcessor) {
|
||||
this.vrPostPublishProcessor = vrPostPublishProcessor;
|
||||
this.threadId = threadId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startDocument() {
|
||||
vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "START"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startElement(String uri, String localName, String qName, Attributes attributes) {
|
||||
|
||||
switch (qName.toLowerCase()) {
|
||||
case "thread":
|
||||
threadTitle = attributes.getValue("title").trim();
|
||||
break;
|
||||
case "post":
|
||||
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
|
||||
postId = attributes.getValue("id").trim();
|
||||
postCounter = Integer.parseInt(attributes.getValue("number").trim());
|
||||
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
|
||||
break;
|
||||
case "image":
|
||||
if (previewCounter++ < 4) {
|
||||
String thumbUrl = Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).orElse(null);
|
||||
if (thumbUrl != null) {
|
||||
previews.add(thumbUrl);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endElement(String uri, String localName, String qName) {
|
||||
switch (qName.toLowerCase()) {
|
||||
case "post":
|
||||
if (imageCount != 0) {
|
||||
vrPostPublishProcessor.onNext(new VRPostParse(
|
||||
threadId,
|
||||
postId,
|
||||
postCounter,
|
||||
postTitle,
|
||||
imageCount,
|
||||
String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
|
||||
previews
|
||||
));
|
||||
}
|
||||
previewCounter = 0;
|
||||
previews = new ArrayList<>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endDocument() {
|
||||
vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "END"));
|
||||
vrPostPublishProcessor.onComplete();
|
||||
}
|
||||
}
|
||||
|
||||
public class VRPostParser {
|
||||
|
||||
private String threadId;
|
||||
private String postId;
|
||||
|
||||
public VRPostParser(String threadId, String postId) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
|
||||
public Post parse() throws PostParseException {
|
||||
|
||||
Post post;
|
||||
logger.info(String.format("Parsing post %s", postId));
|
||||
HttpGet httpGet;
|
||||
try {
|
||||
URIBuilder uriBuilder = new URIBuilder(VR_API);
|
||||
uriBuilder.setParameter("p", postId);
|
||||
httpGet = cm.buildHttpGet(uriBuilder.build());
|
||||
if (vipergirlsAuthService.getCookies() != null) {
|
||||
httpGet.addHeader("Cookie", vipergirlsAuthService.getCookies());
|
||||
}
|
||||
} catch (URISyntaxException e) {
|
||||
throw new PostParseException(e);
|
||||
}
|
||||
VRPostHandler handler = new VRPostHandler(threadId, postId);
|
||||
HttpClient connection = cm.getClient().build();
|
||||
logger.info(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
|
||||
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
|
||||
post = handler.getParsedPost();
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
logger.error("parsing failed", e);
|
||||
throw new PostParseException(e);
|
||||
}
|
||||
return post;
|
||||
}
|
||||
}
|
||||
|
||||
private class VRPostHandler extends DefaultHandler {
|
||||
|
||||
private final String threadId;
|
||||
private final String postId;
|
||||
private String threadTitle;
|
||||
|
||||
private int previewCounter = 0;
|
||||
private int index = 0;
|
||||
|
||||
private String postTitle;
|
||||
private int imageCount;
|
||||
|
||||
private List<String> previews = new ArrayList<>();
|
||||
private List<Image> images = new ArrayList<>();
|
||||
@Getter
|
||||
private Post parsedPost;
|
||||
|
||||
public VRPostHandler(String threadId, String postId) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startDocument() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startElement(String uri, String localName, String qName, Attributes attributes) {
|
||||
|
||||
switch (qName.toLowerCase()) {
|
||||
case "thread":
|
||||
threadTitle = attributes.getValue("title").trim();
|
||||
break;
|
||||
case "post":
|
||||
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
|
||||
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
|
||||
break;
|
||||
case "image":
|
||||
index++;
|
||||
if (previewCounter++ < 4) {
|
||||
String thumbUrl = Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).orElse(null);
|
||||
if (thumbUrl != null) {
|
||||
previews.add(thumbUrl);
|
||||
}
|
||||
}
|
||||
|
||||
String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
|
||||
if (mainUrl != null) {
|
||||
Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
|
||||
if (foundHost != null) {
|
||||
logger.info(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
|
||||
images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
|
||||
} else {
|
||||
logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endElement(String uri, String localName, String qName) {
|
||||
switch (qName.toLowerCase()) {
|
||||
case "post":
|
||||
if (imageCount != 0) {
|
||||
HashMap<String, Object> metadata = new HashMap<>();
|
||||
metadata.put("PREVIEWS", previews);
|
||||
parsedPost = new Post(
|
||||
postTitle,
|
||||
String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
|
||||
images,
|
||||
metadata,
|
||||
postId,
|
||||
threadId
|
||||
);
|
||||
}
|
||||
index = 0;
|
||||
previewCounter = 0;
|
||||
previews = new ArrayList<>();
|
||||
images = new ArrayList<>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
public VRThreadParser createVRThreadParser(String threadId) {
|
||||
return new VRThreadParser(threadId, cm, vipergirlsAuthService);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import lombok.Getter;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import net.jodah.failsafe.RetryPolicy;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.utils.URIBuilder;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.xml.sax.Attributes;
|
||||
import org.xml.sax.helpers.DefaultHandler;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static tn.mnlr.vripper.services.PostParser.VR_API;
|
||||
|
||||
class VRPostParser {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VRPostParser.class);
|
||||
|
||||
private static SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||
|
||||
private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
|
||||
.handleIf(e -> e instanceof IOException)
|
||||
.withBackoff(5, 30, ChronoUnit.SECONDS)
|
||||
.withMaxRetries(4)
|
||||
.abortOn(InterruptedException.class)
|
||||
.onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
|
||||
|
||||
private final String threadId;
|
||||
private final String postId;
|
||||
private final ConnectionManager cm;
|
||||
private final VipergirlsAuthService vipergirlsAuthService;
|
||||
private final List<Host> supportedHosts;
|
||||
|
||||
VRPostParser(String threadId, String postId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, List<Host> supportedHosts) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
this.cm = cm;
|
||||
this.vipergirlsAuthService = vipergirlsAuthService;
|
||||
this.supportedHosts = supportedHosts;
|
||||
}
|
||||
|
||||
public Post parse() throws PostParseException {
|
||||
|
||||
logger.debug(String.format("Parsing post %s", postId));
|
||||
HttpGet httpGet;
|
||||
try {
|
||||
URIBuilder uriBuilder = new URIBuilder(VR_API);
|
||||
uriBuilder.setParameter("p", postId);
|
||||
httpGet = cm.buildHttpGet(uriBuilder.build());
|
||||
} catch (URISyntaxException e) {
|
||||
throw new PostParseException(e);
|
||||
}
|
||||
VRPostHandler handler = new VRPostHandler(threadId, postId, supportedHosts);
|
||||
AtomicReference<Throwable> thr = new AtomicReference<>();
|
||||
logger.debug(String.format("Requesting %s", httpGet));
|
||||
Post post = Failsafe.with(retryPolicy)
|
||||
.onFailure(e -> thr.set(e.getFailure()))
|
||||
.get(() -> {
|
||||
HttpClient connection = cm.getClient().build();
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
|
||||
try {
|
||||
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
|
||||
return handler.getParsedPost();
|
||||
} catch (Exception e) {
|
||||
throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
}
|
||||
}
|
||||
});
|
||||
if (thr.get() != null || post == null) {
|
||||
logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
|
||||
throw new PostParseException(thr.get());
|
||||
}
|
||||
return post;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
class VRPostParseState extends VRPostState {
|
||||
|
||||
private final String type = "postParse";
|
||||
private String postId;
|
||||
private int number;
|
||||
private String title;
|
||||
private int imageCount;
|
||||
private String url;
|
||||
private List<String> previews;
|
||||
|
||||
|
||||
VRPostParseState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews) {
|
||||
super(threadId);
|
||||
this.postId = postId;
|
||||
this.number = number;
|
||||
this.title = title;
|
||||
this.imageCount = imageCount;
|
||||
this.previews = previews;
|
||||
this.url = url;
|
||||
}
|
||||
}
|
||||
|
||||
class VRPostHandler extends DefaultHandler {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VRPostHandler.class);
|
||||
|
||||
private final List<Host> supportedHosts;
|
||||
|
||||
private final String threadId;
|
||||
private final String postId;
|
||||
private String threadTitle;
|
||||
|
||||
private int previewCounter = 0;
|
||||
private int index = 0;
|
||||
|
||||
private String postTitle;
|
||||
private int imageCount;
|
||||
|
||||
private String forum;
|
||||
|
||||
private List<String> previews = new ArrayList<>();
|
||||
private List<Image> images = new ArrayList<>();
|
||||
@Getter
|
||||
private Post parsedPost;
|
||||
|
||||
VRPostHandler(String threadId, String postId, List<Host> supportedHosts) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
this.supportedHosts = supportedHosts;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startDocument() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startElement(String uri, String localName, String qName, Attributes attributes) {
|
||||
|
||||
switch (qName.toLowerCase()) {
|
||||
case "forum":
|
||||
forum = attributes.getValue("title").trim();
|
||||
break;
|
||||
case "thread":
|
||||
threadTitle = attributes.getValue("title").trim();
|
||||
break;
|
||||
case "post":
|
||||
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
|
||||
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
|
||||
break;
|
||||
case "image":
|
||||
index++;
|
||||
if (previewCounter++ < 4) {
|
||||
Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(thumbUrl -> previews.add(thumbUrl));
|
||||
}
|
||||
|
||||
String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
|
||||
if (mainUrl != null) {
|
||||
Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
|
||||
if (foundHost != null) {
|
||||
logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
|
||||
images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
|
||||
} else {
|
||||
logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endElement(String uri, String localName, String qName) {
|
||||
if ("post".equals(qName.toLowerCase())) {
|
||||
if (imageCount != 0) {
|
||||
HashMap<String, Object> metadata = new HashMap<>();
|
||||
metadata.put("PREVIEWS", previews);
|
||||
parsedPost = new Post(
|
||||
postTitle,
|
||||
String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
|
||||
images,
|
||||
metadata,
|
||||
postId,
|
||||
threadId,
|
||||
threadTitle,
|
||||
forum
|
||||
);
|
||||
}
|
||||
index = 0;
|
||||
previewCounter = 0;
|
||||
previews = new ArrayList<>();
|
||||
images = new ArrayList<>();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public abstract class VRPostState {
|
||||
private final String threadId;
|
||||
|
||||
VRPostState(String threadId) {
|
||||
this.threadId = threadId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import io.reactivex.processors.PublishProcessor;
|
||||
import lombok.Getter;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import net.jodah.failsafe.RetryPolicy;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.utils.URIBuilder;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.xml.sax.Attributes;
|
||||
import org.xml.sax.helpers.DefaultHandler;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static tn.mnlr.vripper.services.PostParser.VR_API;
|
||||
|
||||
public class VRThreadParser {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VRThreadParser.class);
|
||||
|
||||
private static RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
|
||||
.handleIf(e -> e instanceof IOException)
|
||||
.withBackoff(5, 30, ChronoUnit.SECONDS)
|
||||
.withMaxRetries(4)
|
||||
.abortOn(InterruptedException.class)
|
||||
.onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
|
||||
private static SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||
@Getter
|
||||
private final PublishProcessor<VRPostState> postPublishProcessor = PublishProcessor.create();
|
||||
private final String threadId;
|
||||
private final ConnectionManager cm;
|
||||
private final VipergirlsAuthService vipergirlsAuthService;
|
||||
|
||||
|
||||
VRThreadParser(String threadId, ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService) {
|
||||
this.threadId = threadId;
|
||||
this.cm = cm;
|
||||
this.vipergirlsAuthService = vipergirlsAuthService;
|
||||
}
|
||||
|
||||
public Void parse() throws PostParseException {
|
||||
|
||||
logger.debug(String.format("Parsing thread %s", threadId));
|
||||
HttpGet httpGet;
|
||||
try {
|
||||
URIBuilder uriBuilder = new URIBuilder(VR_API);
|
||||
uriBuilder.setParameter("t", threadId);
|
||||
httpGet = cm.buildHttpGet(uriBuilder.build());
|
||||
} catch (URISyntaxException e) {
|
||||
throw new PostParseException(e);
|
||||
}
|
||||
|
||||
VRThreadHandler handler = new VRThreadHandler(threadId, postPublishProcessor);
|
||||
AtomicReference<Throwable> thr = new AtomicReference<>();
|
||||
logger.debug(String.format("Requesting %s", httpGet));
|
||||
Failsafe.with(retryPolicy)
|
||||
.onFailure(e -> thr.set(e.getFailure()))
|
||||
.get(() -> {
|
||||
HttpClient connection = cm.getClient().build();
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
|
||||
try {
|
||||
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
|
||||
} catch (Exception e) {
|
||||
throw new PostParseException(String.format("Failed to parse thread %s", threadId), e);
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
if (thr.get() != null) {
|
||||
logger.error(String.format("parsing failed for thread %s", threadId), thr.get());
|
||||
throw new PostParseException(thr.get());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
class VRThreadHandler extends DefaultHandler {
|
||||
|
||||
private final String threadId;
|
||||
private final PublishProcessor<VRPostState> vrPostPublishProcessor;
|
||||
private String threadTitle;
|
||||
private String postId;
|
||||
private String postTitle;
|
||||
private int imageCount;
|
||||
private int postCounter;
|
||||
private int previewCounter = 0;
|
||||
private List<String> previews = new ArrayList<>();
|
||||
|
||||
VRThreadHandler(String threadId, PublishProcessor<VRPostState> vrPostPublishProcessor) {
|
||||
this.vrPostPublishProcessor = vrPostPublishProcessor;
|
||||
this.threadId = threadId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startDocument() {
|
||||
vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "START"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startElement(String uri, String localName, String qName, Attributes attributes) {
|
||||
|
||||
switch (qName.toLowerCase()) {
|
||||
case "thread":
|
||||
threadTitle = attributes.getValue("title").trim();
|
||||
break;
|
||||
case "post":
|
||||
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
|
||||
postId = attributes.getValue("id").trim();
|
||||
postCounter = Integer.parseInt(attributes.getValue("number").trim());
|
||||
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
|
||||
break;
|
||||
case "image":
|
||||
if (previewCounter++ < 4) {
|
||||
Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(thumbUrl -> previews.add(thumbUrl));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endElement(String uri, String localName, String qName) {
|
||||
if ("post".equals(qName.toLowerCase())) {
|
||||
if (imageCount != 0) {
|
||||
vrPostPublishProcessor.onNext(new VRPostParseState(
|
||||
threadId,
|
||||
postId,
|
||||
postCounter,
|
||||
postTitle,
|
||||
imageCount,
|
||||
String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId),
|
||||
previews
|
||||
));
|
||||
}
|
||||
previewCounter = 0;
|
||||
previews = new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endDocument() {
|
||||
vrPostPublishProcessor.onNext(new VRThreadParseState(threadId, "END"));
|
||||
vrPostPublishProcessor.onComplete();
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
class VRThreadParseState extends VRPostState {
|
||||
|
||||
private final String type = "threadParseState";
|
||||
private final String state;
|
||||
|
||||
VRThreadParseState(String threadId, String state) {
|
||||
super(threadId);
|
||||
this.state = state;
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,9 @@ import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.cookie.Cookie;
|
||||
import org.apache.http.impl.client.BasicCookieStore;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
@@ -18,6 +21,7 @@ import org.w3c.dom.Document;
|
||||
import tn.mnlr.vripper.VripperApplication;
|
||||
import tn.mnlr.vripper.exception.VripperException;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
@@ -42,7 +46,7 @@ public class VipergirlsAuthService {
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
@Getter
|
||||
private String cookies;
|
||||
private HttpClientContext context = HttpClientContext.create();
|
||||
|
||||
private boolean authenticated = false;
|
||||
|
||||
@@ -52,6 +56,11 @@ public class VipergirlsAuthService {
|
||||
@Getter
|
||||
private PublishProcessor<String> loggedInUser = PublishProcessor.create();
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() throws InterruptedException {
|
||||
logger.info("Shutting down VipergirlsAuthService");
|
||||
@@ -65,8 +74,8 @@ public class VipergirlsAuthService {
|
||||
authenticated = false;
|
||||
|
||||
if(!appSettingsService.isVLogin()) {
|
||||
logger.warn("Authentication option is disabled");
|
||||
cookies = null;
|
||||
logger.debug("Authentication option is disabled");
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
loggedInUser.onNext(loggedUser);
|
||||
return;
|
||||
@@ -77,7 +86,7 @@ public class VipergirlsAuthService {
|
||||
|
||||
if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
|
||||
logger.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
|
||||
cookies = null;
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
loggedInUser.onNext(loggedUser);
|
||||
return;
|
||||
@@ -97,7 +106,7 @@ public class VipergirlsAuthService {
|
||||
try {
|
||||
postAuth.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
cookies = null;
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
loggedInUser.onNext(loggedUser);
|
||||
throw new VripperException(e);
|
||||
@@ -108,28 +117,19 @@ public class VipergirlsAuthService {
|
||||
|
||||
CloseableHttpClient client = cm.getClient().build();
|
||||
|
||||
try (CloseableHttpResponse response = client.execute(postAuth)) {
|
||||
try (CloseableHttpResponse response = client.execute(postAuth, context)) {
|
||||
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new VripperException(String.format("Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Arrays.asList(response.getHeaders("set-cookie"))
|
||||
.stream()
|
||||
.map(e -> e.getValue())
|
||||
.map(e -> e.split(";")[0])
|
||||
.filter(e -> e.startsWith("vg_lastactivity") || e.startsWith("vg_userid") || e.startsWith("vg_password"))
|
||||
.forEach(e -> sb.append(e).append(";"));
|
||||
String cookies = sb.toString();
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
logger.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
if(!cookies.contains("vg_userid")) {
|
||||
if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) {
|
||||
throw new VripperException("Failed to authenticate user with ViperRipper");
|
||||
}
|
||||
this.cookies = cookies;
|
||||
} catch (Exception e) {
|
||||
cookies = null;
|
||||
context.getCookieStore().clear();
|
||||
loggedUser = "";
|
||||
loggedInUser.onNext(loggedUser);
|
||||
if (e instanceof VripperException) {
|
||||
@@ -144,14 +144,14 @@ public class VipergirlsAuthService {
|
||||
loggedInUser.onNext(loggedUser);
|
||||
}
|
||||
|
||||
public void leaveThanks(String postUrl, String postId) {
|
||||
void leaveThanks(String postUrl, String postId) {
|
||||
VripperApplication.commonExecutor.submit(() -> {
|
||||
if (!appSettingsService.isVLogin()) {
|
||||
logger.warn("Authentication with ViperGirls option is disabled");
|
||||
logger.debug("Authentication with ViperGirls option is disabled");
|
||||
return;
|
||||
}
|
||||
if (!appSettingsService.isVThanks()) {
|
||||
logger.warn("Leave thanks option is disabled");
|
||||
logger.debug("Leave thanks option is disabled");
|
||||
return;
|
||||
}
|
||||
if (!authenticated) {
|
||||
@@ -168,16 +168,15 @@ public class VipergirlsAuthService {
|
||||
|
||||
private String getSecurityToken(String url) throws VripperException {
|
||||
|
||||
String securityToken = "";
|
||||
String securityToken;
|
||||
|
||||
HttpGet httpGet = cm.buildHttpGet(url);
|
||||
httpGet.addHeader("Referer", "https://vipergirls.to/");
|
||||
httpGet.addHeader("Host", "vipergirls.to");
|
||||
httpGet.addHeader("Cookie", cookies);
|
||||
|
||||
CloseableHttpClient client = cm.getClient().build();
|
||||
|
||||
try (CloseableHttpResponse response = client.execute(httpGet)) {
|
||||
try (CloseableHttpResponse response = client.execute(httpGet, context)) {
|
||||
|
||||
String postPage = EntityUtils.toString(response.getEntity());
|
||||
Document document = htmlProcessorService.clean(postPage);
|
||||
@@ -189,7 +188,7 @@ public class VipergirlsAuthService {
|
||||
.getTextContent()
|
||||
.trim();
|
||||
|
||||
securityToken = Arrays.asList(thanksUrl.split("&")).stream()
|
||||
securityToken = Arrays.stream(thanksUrl.split("&"))
|
||||
.filter(v -> v.startsWith("securitytoken"))
|
||||
.findAny()
|
||||
.orElse("")
|
||||
@@ -218,11 +217,10 @@ public class VipergirlsAuthService {
|
||||
|
||||
postThanks.addHeader("Referer", "https://vipergirls.to/");
|
||||
postThanks.addHeader("Host", "vipergirls.to");
|
||||
postThanks.addHeader("Cookie", cookies);
|
||||
|
||||
CloseableHttpClient client = cm.getClient().build();
|
||||
|
||||
try (CloseableHttpResponse response = client.execute(postThanks)) {
|
||||
try (CloseableHttpResponse response = client.execute(postThanks, context)) {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
throw new VripperException(e);
|
||||
|
||||
@@ -2,7 +2,6 @@ package tn.mnlr.vripper.services;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
|
||||
import javax.xml.xpath.XPath;
|
||||
@@ -12,20 +11,8 @@ import javax.xml.xpath.XPathFactory;
|
||||
@Service
|
||||
public class XpathService {
|
||||
|
||||
private XpathService() {
|
||||
|
||||
}
|
||||
|
||||
private XPath xpath = XPathFactory.newInstance().newXPath();
|
||||
|
||||
public NodeList getAsNodeList(Node source, String xpathExpression) throws XpathException {
|
||||
try {
|
||||
return (NodeList) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET);
|
||||
} catch (Exception e) {
|
||||
throw new XpathException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public Node getAsNode(Node source, String xpathExpression) throws XpathException {
|
||||
try {
|
||||
return (Node) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE);
|
||||
|
||||
+108
-104
@@ -1,7 +1,9 @@
|
||||
package tn.mnlr.vripper.web.restendpoints;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.ToString;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -17,6 +19,7 @@ import tn.mnlr.vripper.services.PathService;
|
||||
import tn.mnlr.vripper.services.PostParser;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -27,8 +30,7 @@ public class PostRestEndpoint {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
|
||||
|
||||
|
||||
private static final Pattern VG_URL_PATTERN = Pattern.compile("https:\\/\\/vipergirls\\.to\\/threads\\/(\\d+)((.*p=)(\\d+))?");
|
||||
private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?");
|
||||
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
@@ -56,16 +58,16 @@ public class PostRestEndpoint {
|
||||
@PostMapping("/post")
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public ResponseEntity processPost(@RequestBody ThreadUrl url) throws Exception {
|
||||
logger.info(String.format("Starting to process thread: %s", url.url));
|
||||
if (url.url == null || url.url.isEmpty()) {
|
||||
logger.debug(String.format("Starting to process thread: %s", url.getUrl()));
|
||||
if (url.getUrl() == null || url.getUrl().isEmpty()) {
|
||||
return new ResponseEntity<>("Failed to process empty request", HttpStatus.BAD_REQUEST);
|
||||
} else if (!url.url.startsWith("https://vipergirls.to")) {
|
||||
return new ResponseEntity<>("ViperGirls only links are supported", HttpStatus.BAD_REQUEST);
|
||||
} else if (!url.getUrl().startsWith("https://vipergirls.to")) {
|
||||
return new ResponseEntity<>("ViperGirls links only are supported", HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
String threadId, postId;
|
||||
try {
|
||||
Matcher m = VG_URL_PATTERN.matcher(url.url);
|
||||
Matcher m = VG_URL_PATTERN.matcher(url.getUrl());
|
||||
if (m.find()) {
|
||||
threadId = m.group(1);
|
||||
postId = m.group(4);
|
||||
@@ -80,8 +82,10 @@ public class PostRestEndpoint {
|
||||
|
||||
@PostMapping("/post/restart")
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public void restartPost(@RequestBody PostId postId) throws Exception {
|
||||
downloadQ.restart(postId.getPostId());
|
||||
public synchronized void restartPost(@RequestBody @NonNull List<PostId> postIds) throws Exception {
|
||||
for (PostId postId : postIds) {
|
||||
downloadQ.restart(postId.getPostId());
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("/post/add")
|
||||
@@ -90,9 +94,9 @@ public class PostRestEndpoint {
|
||||
for (PostToAdd post : posts) {
|
||||
VripperApplication.commonExecutor.submit(() -> {
|
||||
try {
|
||||
postParser.addPost(post.postId, post.threadId);
|
||||
postParser.addPost(post.getPostId(), post.getThreadId());
|
||||
} catch (PostParseException e) {
|
||||
logger.error(String.format("Failed to add post %s", post.postId), e);
|
||||
logger.error(String.format("Failed to add post %s", post.getPostId()), e);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -113,8 +117,10 @@ public class PostRestEndpoint {
|
||||
|
||||
@PostMapping("/post/stop")
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public void stop(@RequestBody PostId postId) {
|
||||
downloadQ.stop(postId.getPostId());
|
||||
public synchronized void stop(@RequestBody @NonNull List<PostId> postIds) {
|
||||
for (PostId postId : postIds) {
|
||||
downloadQ.stop(postId.getPostId());
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("/post/stop/all")
|
||||
@@ -125,10 +131,14 @@ public class PostRestEndpoint {
|
||||
|
||||
@PostMapping("/post/remove")
|
||||
@ResponseStatus(value = HttpStatus.OK)
|
||||
public ResponseEntity remove(@RequestBody PostId postId) {
|
||||
downloadQ.stop(postId.getPostId());
|
||||
appStateService.remove(postId.getPostId());
|
||||
return ResponseEntity.ok(new RemoveResult(postId.getPostId()));
|
||||
public synchronized ResponseEntity remove(@RequestBody @NonNull List<PostId> postIds) {
|
||||
List<RemoveResult> result = new ArrayList<>();
|
||||
for (PostId postId : postIds) {
|
||||
downloadQ.stop(postId.getPostId());
|
||||
appStateService.remove(postId.getPostId());
|
||||
result.add(new RemoveResult(postId.getPostId()));
|
||||
}
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
|
||||
@PostMapping("/post/clear/all")
|
||||
@@ -142,93 +152,87 @@ public class PostRestEndpoint {
|
||||
public ResponseEntity removeAll() {
|
||||
return ResponseEntity.ok(new RemoveAllResult(appStateService.removeAll()));
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@ToString
|
||||
private static class ThreadUrl {
|
||||
private String url;
|
||||
}
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class ThreadUrl {
|
||||
private String url;
|
||||
|
||||
@Getter
|
||||
private static class PostId {
|
||||
private String postId;
|
||||
}
|
||||
|
||||
@Getter
|
||||
private static class PairThreadIdPostId {
|
||||
private String threadId;
|
||||
private String postId;
|
||||
|
||||
public PairThreadIdPostId(String threadId, String postId) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
private static class PostToAdd {
|
||||
private String threadId;
|
||||
private String postId;
|
||||
}
|
||||
|
||||
@Getter
|
||||
private static class ParseResult {
|
||||
|
||||
private List<PostResult> posts;
|
||||
private int count;
|
||||
private String threadId;
|
||||
|
||||
public ParseResult(List<PostResult> posts, int count, String threadId) {
|
||||
this.posts = posts;
|
||||
this.count = count;
|
||||
this.threadId = threadId;
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class PostResult {
|
||||
|
||||
private String title;
|
||||
private int counter;
|
||||
private String url;
|
||||
private String postId;
|
||||
private List<String> previews;
|
||||
|
||||
public PostResult(String title, int counter, String url, String postId, List<String> previews) {
|
||||
this.title = title;
|
||||
this.counter = counter;
|
||||
this.url = url;
|
||||
this.postId = postId;
|
||||
this.previews = previews;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
private static class RemoveAllResult {
|
||||
private List<String> postIds;
|
||||
private int removed;
|
||||
|
||||
RemoveAllResult(List<String> postIds) {
|
||||
this.removed = postIds.size();
|
||||
this.postIds = postIds;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
private static class RemoveResult {
|
||||
private String postId;
|
||||
|
||||
RemoveResult(String postId) {
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
private static class DownloadPath {
|
||||
private String path;
|
||||
|
||||
DownloadPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
public ThreadUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class PostId {
|
||||
private String postId;
|
||||
|
||||
public PostId(String postId) {
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class PairThreadIdPostId {
|
||||
private String threadId;
|
||||
private String postId;
|
||||
|
||||
PairThreadIdPostId(String threadId, String postId) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class PostToAdd {
|
||||
private String threadId;
|
||||
private String postId;
|
||||
|
||||
public PostToAdd(String threadId, String postId) {
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class RemoveAllResult {
|
||||
private List<String> postIds;
|
||||
private int removed;
|
||||
|
||||
RemoveAllResult(List<String> postIds) {
|
||||
this.removed = postIds.size();
|
||||
this.postIds = postIds;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class RemoveResult {
|
||||
private String postId;
|
||||
|
||||
RemoveResult(String postId) {
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
class DownloadPath {
|
||||
private String path;
|
||||
|
||||
DownloadPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
}
|
||||
+12
-4
@@ -54,7 +54,9 @@ public class SettingsRestEndpoint {
|
||||
try {
|
||||
this.settings.check(settings);
|
||||
} catch (ValidationException e) {
|
||||
return new ResponseEntity(new Response(e.getMessage()), HttpStatus.BAD_REQUEST);
|
||||
return ResponseEntity
|
||||
.status(HttpStatus.BAD_REQUEST)
|
||||
.body(new Response(e.getMessage()));
|
||||
}
|
||||
this.settings.setDownloadPath(settings.getDownloadPath());
|
||||
this.settings.setMaxThreads(settings.getMaxThreads());
|
||||
@@ -74,6 +76,9 @@ public class SettingsRestEndpoint {
|
||||
}
|
||||
this.settings.setDesktopClipboard(settings.isDesktopClipboard());
|
||||
this.settings.setForceOrder(settings.isForceOrder());
|
||||
this.settings.setSubLocation(settings.isSubLocation());
|
||||
this.settings.setThreadSubLocation(settings.isThreadSubLocation());
|
||||
this.settings.setClearCompleted(settings.isClearCompleted());
|
||||
|
||||
this.settings.save();
|
||||
|
||||
@@ -95,15 +100,18 @@ public class SettingsRestEndpoint {
|
||||
settings.getVPassword(),
|
||||
settings.isVThanks(),
|
||||
settings.isDesktopClipboard(),
|
||||
settings.isForceOrder()
|
||||
settings.isForceOrder(),
|
||||
settings.isSubLocation(),
|
||||
settings.isThreadSubLocation(),
|
||||
settings.isClearCompleted()
|
||||
);
|
||||
}
|
||||
|
||||
@Getter
|
||||
class Response {
|
||||
static class Response {
|
||||
String message;
|
||||
|
||||
public Response(String message) {
|
||||
Response(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
}
|
||||
|
||||
+37
-33
@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.schedulers.Schedulers;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -20,7 +22,7 @@ import tn.mnlr.vripper.entities.mixin.ui.PostUIMixin;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@@ -88,42 +90,42 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
subscribeForPostDetails(session, wsMessage.getPayload());
|
||||
break;
|
||||
case POST_DETAILS_UNSUB:
|
||||
logger.info(String.format("Client %s unsubscribed from post details", session.getId()));
|
||||
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
logger.debug(String.format("Client %s unsubscribed from post details", session.getId()));
|
||||
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
break;
|
||||
case THREAD_PARSING_UNSUB:
|
||||
logger.info(String.format("Client %s unsubscribed from thread parsing", session.getId()));
|
||||
Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
logger.debug(String.format("Client %s unsubscribed from thread parsing", session.getId()));
|
||||
Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
|
||||
break;
|
||||
case POSTS_UNSUB:
|
||||
logger.info(String.format("Client %s unsubscribed from posts", session.getId()));
|
||||
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
logger.debug(String.format("Client %s unsubscribed from posts", session.getId()));
|
||||
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
break;
|
||||
case GLOBAL_STATE_UNSUB:
|
||||
logger.info(String.format("Client %s unsubscribed from global state", session.getId()));
|
||||
Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
logger.debug(String.format("Client %s unsubscribed from global state", session.getId()));
|
||||
Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
break;
|
||||
case SPEED_UNSUB:
|
||||
logger.info(String.format("Client %s unsubscribed from download speed info", session.getId()));
|
||||
Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
logger.debug(String.format("Client %s unsubscribed from download speed info", session.getId()));
|
||||
Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
break;
|
||||
case USER_UNSUB:
|
||||
logger.info(String.format("Client %s unsubscribed from user info", session.getId()));
|
||||
Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
logger.debug(String.format("Client %s unsubscribed from user info", session.getId()));
|
||||
Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void subscribeForGlobalState(WebSocketSession session) {
|
||||
|
||||
logger.info(String.format("Client %s subscribed for global state", session.getId()));
|
||||
logger.debug(String.format("Client %s subscribed for global state", session.getId()));
|
||||
if (stateSubscriptions.containsKey(session.getId())) {
|
||||
stateSubscriptions.get(session.getId()).dispose();
|
||||
}
|
||||
|
||||
try {
|
||||
send(session, new TextMessage(om.writeValueAsString(Arrays.asList(globalStateService.getCurrentState()))));
|
||||
send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(globalStateService.getCurrentState()))));
|
||||
} catch (Exception e) {
|
||||
logger.error("Unexpected error occurred", e);
|
||||
}
|
||||
@@ -143,13 +145,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
|
||||
private void subscribeForSpeed(WebSocketSession session) {
|
||||
|
||||
logger.info(String.format("Client %s subscribed for download speed info", session.getId()));
|
||||
logger.debug(String.format("Client %s subscribed for download speed info", session.getId()));
|
||||
if (downloadSpeedSubscriptions.containsKey(session.getId())) {
|
||||
downloadSpeedSubscriptions.get(session.getId()).dispose();
|
||||
}
|
||||
|
||||
try {
|
||||
send(session, new TextMessage(om.writeValueAsString(Arrays.asList(new DownloadSpeed(0)))));
|
||||
send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new DownloadSpeed(0)))));
|
||||
} catch (Exception e) {
|
||||
logger.error("Unexpected error occurred", e);
|
||||
}
|
||||
@@ -170,13 +172,13 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
|
||||
private void subscribeForUser(WebSocketSession session) {
|
||||
|
||||
logger.info(String.format("Client %s subscribed for user info", session.getId()));
|
||||
logger.debug(String.format("Client %s subscribed for user info", session.getId()));
|
||||
if (userSubscriptions.containsKey(session.getId())) {
|
||||
userSubscriptions.get(session.getId()).dispose();
|
||||
}
|
||||
|
||||
try {
|
||||
send(session, new TextMessage(om.writeValueAsString(Arrays.asList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
|
||||
send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
|
||||
} catch (Exception e) {
|
||||
logger.error("Unexpected error occurred", e);
|
||||
}
|
||||
@@ -185,7 +187,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
vipergirlsAuthService.getLoggedInUser()
|
||||
.onBackpressureBuffer()
|
||||
.observeOn(Schedulers.io())
|
||||
.map(e -> Arrays.asList(new LoggedUser(e)))
|
||||
.map(e -> Collections.singletonList(new LoggedUser(e)))
|
||||
.map(om::writeValueAsString)
|
||||
.map(TextMessage::new)
|
||||
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
|
||||
@@ -194,7 +196,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
|
||||
private void subscribeForPosts(WebSocketSession session) {
|
||||
|
||||
logger.info(String.format("Client %s subscribed for posts", session.getId()));
|
||||
logger.debug(String.format("Client %s subscribed for posts", session.getId()));
|
||||
if (postsSubscriptions.containsKey(session.getId())) {
|
||||
postsSubscriptions.get(session.getId()).dispose();
|
||||
}
|
||||
@@ -220,7 +222,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
|
||||
private void subscribeForThreadParsing(WebSocketSession session, String threadId) {
|
||||
|
||||
logger.info(String.format("Client %s subscribed for thread parsing with threadId = %s", session.getId(), threadId));
|
||||
logger.debug(String.format("Client %s subscribed for thread parsing with threadId = %s", session.getId(), threadId));
|
||||
if (vrPostParserSubscriptions.containsKey(session.getId())) {
|
||||
vrPostParserSubscriptions.get(session.getId()).dispose();
|
||||
}
|
||||
@@ -229,7 +231,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
threadParseRequests.get(session.getId()).cancel(true);
|
||||
}
|
||||
|
||||
PostParser.VRThreadParser vrThreadParser = postParser.new VRThreadParser(threadId);
|
||||
VRThreadParser vrThreadParser = postParser.createVRThreadParser(threadId);
|
||||
|
||||
vrPostParserSubscriptions.put(session.getId(), vrThreadParser.getPostPublishProcessor()
|
||||
.onBackpressureBuffer()
|
||||
@@ -246,7 +248,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
|
||||
private void subscribeForPostDetails(WebSocketSession session, String postId) {
|
||||
|
||||
logger.info(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
|
||||
logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
|
||||
if (postDetailsSubscriptions.containsKey(session.getId())) {
|
||||
postDetailsSubscriptions.get(session.getId()).dispose();
|
||||
}
|
||||
@@ -282,24 +284,26 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
|
||||
@Override
|
||||
public void afterConnectionEstablished(WebSocketSession session) {
|
||||
logger.info(String.format("Connection open for client id: %s", session.getId()));
|
||||
logger.debug(String.format("Connection open for client id: %s", session.getId()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
|
||||
|
||||
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(d -> d.dispose());
|
||||
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(vrPostParserSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
|
||||
Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
|
||||
|
||||
logger.info(String.format("Connection closed for client id: %s", session.getId()));
|
||||
logger.debug(String.format("Connection closed for client id: %s", session.getId()));
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
private static class WSMessage {
|
||||
|
||||
private String cmd;
|
||||
@@ -327,7 +331,7 @@ public class WebSocketHandler extends TextWebSocketHandler {
|
||||
private final String type = "user";
|
||||
private String user;
|
||||
|
||||
public LoggedUser(String user) {
|
||||
LoggedUser(String user) {
|
||||
this.user = user;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
base.dir=${user.home}
|
||||
@@ -0,0 +1 @@
|
||||
base.dir=${user.dir}
|
||||
@@ -1,8 +1,10 @@
|
||||
logging.level.org.springframework.web=INFO
|
||||
logging.level.root=INFO
|
||||
logging.level.org.apache.http=INFO
|
||||
logging.file=${user.dir}/vripper.log
|
||||
logging.file=${base.dir}/.vripper/vripper.log
|
||||
server.port=${vripper.server.port:8080}
|
||||
management.endpoints.web.exposure.include=shutdown
|
||||
management.endpoint.shutdown.enabled=true
|
||||
endpoints.shutdown.enabled=true
|
||||
endpoints.shutdown.enabled=true
|
||||
spring.profiles.active=portable
|
||||
#spring.profiles.active=installer
|
||||
Generated
+7
-7
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-ui",
|
||||
"version": "1.6.4",
|
||||
"version": "2.4.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
@@ -3510,9 +3510,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"electron": {
|
||||
"version": "6.0.4",
|
||||
"resolved": "https://registry.npmjs.org/electron/-/electron-6.0.4.tgz",
|
||||
"integrity": "sha512-zrPi36etADOAjxnVX6TxRNKSWaBscMLd9S7AB+qISzI0dnYIDKycHpc2mB+5QWBd/8cR4m/1NLNTqNhX5KKGFg==",
|
||||
"version": "6.0.9",
|
||||
"resolved": "https://registry.npmjs.org/electron/-/electron-6.0.9.tgz",
|
||||
"integrity": "sha512-lFpSmDNyjpvJFwEnK897Soone3DV7D3ASFUb315H2VTVZSbKib9Kbrsovxf4c+e1q5hTdaONsGIm3Lb4CXIW1g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "^10.12.18",
|
||||
@@ -3521,9 +3521,9 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/node": {
|
||||
"version": "10.14.16",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.16.tgz",
|
||||
"integrity": "sha512-/opXIbfn0P+VLt+N8DE4l8Mn8rbhiJgabU96ZJ0p9mxOkIks5gh6RUnpHak7Yh0SFkyjO/ODbxsQQPV2bpMmyA==",
|
||||
"version": "10.14.18",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.18.tgz",
|
||||
"integrity": "sha512-ryO3Q3++yZC/+b8j8BdKd/dn9JlzlHBPdm80656xwYUdmPkpTGTjkAdt6BByiNupGPE8w0FhBgvYy/fX9hRNGQ==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-ui",
|
||||
"version": "1.6.4",
|
||||
"version": "2.4.0",
|
||||
"scripts": {
|
||||
"ng": "ng",
|
||||
"start": "ng serve",
|
||||
@@ -42,7 +42,7 @@
|
||||
"@types/jasminewd2": "~2.0.3",
|
||||
"@types/node": "~8.9.4",
|
||||
"codelyzer": "~4.5.0",
|
||||
"electron": "^6.0.4",
|
||||
"electron": "^6.0.9",
|
||||
"jasmine-core": "~2.99.1",
|
||||
"jasmine-spec-reporter": "~4.2.1",
|
||||
"karma": "^4.2.0",
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>1.6.4</version>
|
||||
<version>2.4.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-ui</artifactId>
|
||||
<name>vripper-ui</name>
|
||||
|
||||
@@ -6,37 +6,9 @@
|
||||
<mat-icon class="overlay-icon">error</mat-icon>
|
||||
<h2>The app seems to be shutdown</h2>
|
||||
</div>
|
||||
<div [ngClass]="{'electron': electronService.isElectronApp}" id="app-container" fxLayout="column" fxLayoutGap="20px">
|
||||
<div [ngClass]="{ electron: electronService.isElectronApp }" fxLayout="column" id="app-container">
|
||||
<div id="app-header" fxFlex="nogrow">
|
||||
<mat-toolbar class="mat-elevation-z8" color="accent">
|
||||
<mat-toolbar-row fxLayout="row" fxLayoutAlign="center center" fxLayoutGap="10px">
|
||||
<div id="left-controls">
|
||||
<button (click)="scan()" class="add-button" color="primary" mat-icon-button>
|
||||
<mat-icon>add</mat-icon>
|
||||
</button>
|
||||
<button (click)="restartAll()" aria-label="Start" mat-icon-button title="Start">
|
||||
<mat-icon>play_arrow</mat-icon>
|
||||
</button>
|
||||
<button (click)="stopAll()" aria-label="Stop" mat-icon-button title="Stop">
|
||||
<mat-icon>stop</mat-icon>
|
||||
</button>
|
||||
<button (click)="clear()" aria-label="Clear completed" mat-icon-button title="Clear completed">
|
||||
<mat-icon>clear_all</mat-icon>
|
||||
</button>
|
||||
<button (click)="remove()" aria-label="Remove All" mat-icon-button title="Remove All">
|
||||
<mat-icon>delete</mat-icon>
|
||||
</button>
|
||||
</div>
|
||||
<span fxFlex="grow" fxLayoutAlign="end center">
|
||||
<p style="font-size: 16px">Logged in as: {{loggedUser.user == null || loggedUser.user === '' ? 'guest': loggedUser.user }}</p>
|
||||
</span>
|
||||
<div id="right-controls">
|
||||
<button (click)="openSettings()" aria-label="settings" mat-icon-button>
|
||||
<mat-icon>menu</mat-icon>
|
||||
</button>
|
||||
</div>
|
||||
</mat-toolbar-row>
|
||||
</mat-toolbar>
|
||||
<app-toolbar></app-toolbar>
|
||||
</div>
|
||||
<div id="app-body" fxFlex="grow">
|
||||
<router-outlet></router-outlet>
|
||||
|
||||
@@ -1,11 +1,3 @@
|
||||
#logo {
|
||||
height: 48px;
|
||||
margin-right: 10px;
|
||||
background-image: url('../assets/logo.png');
|
||||
background-repeat: no-repeat;
|
||||
background-size: 48px 48px;
|
||||
}
|
||||
|
||||
#title {
|
||||
user-select: none
|
||||
}
|
||||
@@ -19,7 +11,3 @@
|
||||
.no-connection {
|
||||
color: #eb6060;
|
||||
}
|
||||
|
||||
.electron {
|
||||
margin-top: 30px;
|
||||
}
|
||||
|
||||
@@ -1,21 +1,11 @@
|
||||
import { AppService } from './app.service';
|
||||
import { ClipboardService } from './clipboard.service';
|
||||
import { ElectronService } from 'ngx-electron';
|
||||
import { SettingsComponent } from './settings/settings.component';
|
||||
import { Component, OnInit, OnDestroy, NgZone, Renderer2, AfterViewInit } from '@angular/core';
|
||||
import { MatDialog, MatSnackBar } from '@angular/material';
|
||||
import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout';
|
||||
import { Observable, Subscription } from 'rxjs';
|
||||
import { Component, OnInit, OnDestroy, NgZone, AfterViewInit, Renderer2 } from '@angular/core';
|
||||
import { MatDialog } from '@angular/material';
|
||||
import { Subscription } from 'rxjs';
|
||||
import { WsConnectionService, WSState } from './ws-connection.service';
|
||||
import { LoggedUser } from './common/logged-user.model';
|
||||
import { WsHandler } from './ws-handler';
|
||||
import { CMD } from './common/cmd.enum';
|
||||
import { WSMessage } from './common/ws-message.model';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { RemoveAllResponse } from './common/remove-all-response.model';
|
||||
import { ServerService } from './server-service';
|
||||
import { ConfirmDialogComponent } from './common/confirmation-component/confirmation-dialog';
|
||||
import { filter, flatMap } from 'rxjs/operators';
|
||||
|
||||
@Component({
|
||||
selector: 'app-root',
|
||||
@@ -25,55 +15,18 @@ import { filter, flatMap } from 'rxjs/operators';
|
||||
export class AppComponent implements OnInit, OnDestroy, AfterViewInit {
|
||||
constructor(
|
||||
public dialog: MatDialog,
|
||||
private breakpointObserver: BreakpointObserver,
|
||||
private ws: WsConnectionService,
|
||||
public electronService: ElectronService,
|
||||
private clipboardService: ClipboardService,
|
||||
private ngZone: NgZone,
|
||||
private httpClient: HttpClient,
|
||||
private serverService: ServerService,
|
||||
private _snackBar: MatSnackBar,
|
||||
private renderer: Renderer2,
|
||||
private appService: AppService
|
||||
) {
|
||||
this.websocketHandlerPromise = this.ws.getConnection();
|
||||
}
|
||||
private appService: AppService,
|
||||
private renderer: Renderer2
|
||||
) {}
|
||||
|
||||
subscriptions: Subscription[] = [];
|
||||
websocketHandlerPromise: Promise<WsHandler>;
|
||||
currentState: WSState;
|
||||
themeLoaded = false;
|
||||
loggedUser: LoggedUser = new LoggedUser(null);
|
||||
isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
|
||||
|
||||
ngAfterViewInit() {
|
||||
this.appService.renderer = this.renderer;
|
||||
}
|
||||
|
||||
scan() {
|
||||
this.appService.scan();
|
||||
}
|
||||
|
||||
openSettings(): void {
|
||||
const dialogRef = this.dialog.open(SettingsComponent, {
|
||||
width: '70%',
|
||||
height: '70%',
|
||||
maxWidth: '100vw',
|
||||
maxHeight: '100vh'
|
||||
});
|
||||
|
||||
const smallDialogSubscription = this.isExtraSmall.subscribe(result => {
|
||||
if (result.matches) {
|
||||
dialogRef.updateSize('100%', '100%');
|
||||
} else {
|
||||
dialogRef.updateSize('70%', '70%');
|
||||
}
|
||||
});
|
||||
|
||||
dialogRef.afterClosed().subscribe(result => {
|
||||
smallDialogSubscription.unsubscribe();
|
||||
});
|
||||
}
|
||||
|
||||
noConnectionState(): boolean {
|
||||
return this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR;
|
||||
@@ -83,20 +36,13 @@ export class AppComponent implements OnInit, OnDestroy, AfterViewInit {
|
||||
return (this.currentState === WSState.INIT || this.currentState === WSState.CONNECTING) && !this.themeLoaded ;
|
||||
}
|
||||
|
||||
ngAfterViewInit() {
|
||||
this.appService.renderer = this.renderer;
|
||||
}
|
||||
|
||||
ngOnInit() {
|
||||
this.websocketHandlerPromise.then((handler: WsHandler) => {
|
||||
console.log('Connecting to user stream');
|
||||
this.subscriptions.push(
|
||||
handler.subscribeForUser((e: LoggedUser[]) => {
|
||||
this.ngZone.run(() => {
|
||||
this.loggedUser = e[0];
|
||||
});
|
||||
})
|
||||
);
|
||||
handler.send(new WSMessage(CMD.USER_SUB.toString()));
|
||||
});
|
||||
this.currentState = WSState.INIT;
|
||||
this.ws.state.subscribe(e => {
|
||||
this.subscriptions.push(this.ws.state.subscribe(e => {
|
||||
this.currentState = e;
|
||||
if (this.currentState === WSState.CLOSE || this.currentState === WSState.ERROR) {
|
||||
this.dialog.closeAll();
|
||||
@@ -106,80 +52,7 @@ export class AppComponent implements OnInit, OnDestroy, AfterViewInit {
|
||||
.loadTheme()
|
||||
.subscribe(() => this.ngZone.run(() => this.themeLoaded = true));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
clear() {
|
||||
this.ngZone.run(() => {
|
||||
this.httpClient.post<RemoveAllResponse>(this.serverService.baseUrl + '/post/clear/all', {}).subscribe(
|
||||
data => {
|
||||
this._snackBar.open(`${data.removed} items cleared`, null, { duration: 5000 });
|
||||
},
|
||||
error => {
|
||||
this._snackBar.open(error.error, null, {
|
||||
duration: 5000
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
remove() {
|
||||
this.ngZone.run(() => {
|
||||
this.dialog
|
||||
.open(ConfirmDialogComponent, {
|
||||
maxHeight: '100vh',
|
||||
maxWidth: '100vw',
|
||||
height: '200px',
|
||||
width: '60%',
|
||||
data: { header: 'Confirmation', content: 'Are you sure you want to remove all items ?' }
|
||||
})
|
||||
.afterClosed()
|
||||
.pipe(
|
||||
filter(e => e === 'yes'),
|
||||
flatMap(e => this.httpClient.post<RemoveAllResponse>(this.serverService.baseUrl + '/post/remove/all', {}))
|
||||
)
|
||||
.subscribe(
|
||||
data => {
|
||||
this._snackBar.open(`${data.removed} items removed`, null, { duration: 5000 });
|
||||
},
|
||||
error => {
|
||||
this._snackBar.open(error.error, null, {
|
||||
duration: 5000
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
stopAll() {
|
||||
this.ngZone.run(() => {
|
||||
this.httpClient.post(this.serverService.baseUrl + '/post/stop/all', {}).subscribe(
|
||||
() => {
|
||||
this._snackBar.open(`Download stopped`, null, { duration: 5000 });
|
||||
},
|
||||
error => {
|
||||
this._snackBar.open(error.error, null, {
|
||||
duration: 5000
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
restartAll() {
|
||||
this.ngZone.run(() => {
|
||||
this.httpClient.post(this.serverService.baseUrl + '/post/restart/all', {}).subscribe(
|
||||
() => {
|
||||
this._snackBar.open(`Download started`, null, { duration: 5000 });
|
||||
},
|
||||
error => {
|
||||
this._snackBar.open(error.error, null, {
|
||||
duration: 5000
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
ngOnDestroy() {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { PostsDataService } from './posts-data.service';
|
||||
import { AppPreviewDirective } from './common/preview-tooltip.directive';
|
||||
import { WsConnectionService } from './ws-connection.service';
|
||||
import { AppService } from './app.service';
|
||||
@@ -31,6 +32,8 @@ import { UrlRendererComponent } from './multi-post/url-renderer.component';
|
||||
import { FilterComponent } from './filter/filter.component';
|
||||
import { ScanComponent } from './scan/scan.component';
|
||||
import { StatusBarComponent } from './status-bar/status-bar.component';
|
||||
import { SelectionService } from './selection-service';
|
||||
import { ToolbarComponent } from './toolbar/toolbar.component';
|
||||
|
||||
@NgModule({
|
||||
declarations: [
|
||||
@@ -49,13 +52,13 @@ import { StatusBarComponent } from './status-bar/status-bar.component';
|
||||
UrlRendererComponent,
|
||||
FilterComponent,
|
||||
ScanComponent,
|
||||
StatusBarComponent
|
||||
StatusBarComponent,
|
||||
ToolbarComponent
|
||||
],
|
||||
entryComponents: [
|
||||
PostDetailComponent,
|
||||
SettingsComponent,
|
||||
ConfirmDialogComponent,
|
||||
// MultiPostComponent,
|
||||
AppPreviewComponent,
|
||||
ScanComponent
|
||||
],
|
||||
@@ -80,7 +83,9 @@ import { StatusBarComponent } from './status-bar/status-bar.component';
|
||||
WsConnectionService,
|
||||
{ provide: HTTP_INTERCEPTORS, useClass: XhrInterceptorService, multi: true },
|
||||
ServerService,
|
||||
SharedService
|
||||
SharedService,
|
||||
SelectionService,
|
||||
PostsDataService
|
||||
],
|
||||
bootstrap: [AppComponent]
|
||||
})
|
||||
|
||||
@@ -6,7 +6,7 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
|
||||
template: `
|
||||
<div class="previews" style="display: inline-block; white-space: nowrap;">
|
||||
<ng-container *ngFor="let link of links">
|
||||
<img
|
||||
<img style="display: inline-block; max-width:150px; min-width:150px; width: auto; height: auto;"
|
||||
[@simpleFadeAnimation]="'in'"
|
||||
class="mat-elevation-z8"
|
||||
[src]="link"
|
||||
@@ -20,7 +20,6 @@ import { trigger, state, style, transition, animate } from '@angular/animations'
|
||||
max-height: 200px;
|
||||
max-width: 200px;
|
||||
margin-left: 5px;
|
||||
border: 1px solid #900000;
|
||||
}
|
||||
`
|
||||
],
|
||||
|
||||
@@ -18,11 +18,13 @@ export class AppPreviewDirective implements OnInit {
|
||||
ngOnInit() {
|
||||
const positionStrategy = this.overlayPositionBuilder
|
||||
.flexibleConnectedTo(this.elementRef)
|
||||
.withPush(true)
|
||||
.withGrowAfterOpen(true)
|
||||
.withPositions([{
|
||||
originX: 'center',
|
||||
originY: 'top',
|
||||
overlayX: 'center',
|
||||
overlayY: 'bottom',
|
||||
originX: 'start',
|
||||
originY: 'bottom',
|
||||
overlayX: 'start',
|
||||
overlayY: 'top',
|
||||
}]);
|
||||
this.overlayRef = this.overlay.create({ positionStrategy });
|
||||
}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { FilterComponent } from './filter.component';
|
||||
|
||||
describe('FilterComponent', () => {
|
||||
let component: FilterComponent;
|
||||
let fixture: ComponentFixture<FilterComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ FilterComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(FilterComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,11 +1,3 @@
|
||||
<div *ngIf="loading" class="overlay loading" fxLayout="row" fxLayoutAlign="center center">
|
||||
<mat-progress-spinner [mode]="spinnerMode" [value]="spinnerValue" style="position: absolute;"></mat-progress-spinner>
|
||||
</div>
|
||||
<div fxLayout="column" fxLayoutGap="10px" style="height: 100%">
|
||||
<div [fxFlex]="electronService.isElectronApp ? '0 0 calc(100% - 117px)' : '0 0 calc(100% - 90px)'">
|
||||
<app-posts
|
||||
[ngStyle]="{ height: electronService.isElectronApp ? 'calc(100% - 37px)' : '100%' }"
|
||||
style="width: 100%;"
|
||||
></app-posts>
|
||||
</div>
|
||||
<div style="height: calc(100% - 20px)">
|
||||
<app-posts style="width: 100%; height: 100%;"></app-posts>
|
||||
</div>
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { HomeComponent } from './home.component';
|
||||
|
||||
describe('HomeComponent', () => {
|
||||
let component: HomeComponent;
|
||||
let fixture: ComponentFixture<HomeComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ HomeComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(HomeComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,38 +1,25 @@
|
||||
import { AppService } from './../app.service';
|
||||
import { ElectronService } from 'ngx-electron';
|
||||
import { ClipboardService } from './../clipboard.service';
|
||||
import { Component, OnInit, OnDestroy, NgZone } from '@angular/core';
|
||||
import { Component, OnInit, OnDestroy, NgZone, ChangeDetectionStrategy } from '@angular/core';
|
||||
import { MatDialog, MatSnackBar } from '@angular/material';
|
||||
import { WsConnectionService } from '../ws-connection.service';
|
||||
import { WsHandler } from '../ws-handler';
|
||||
import { Subscription } from 'rxjs';
|
||||
|
||||
@Component({
|
||||
selector: 'app-home',
|
||||
templateUrl: './home.component.html',
|
||||
styleUrls: ['./home.component.scss']
|
||||
styleUrls: ['./home.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class HomeComponent implements OnInit, OnDestroy {
|
||||
|
||||
loading = false;
|
||||
spinnerMode = 'indeterminate';
|
||||
spinnerValue = 0;
|
||||
|
||||
parseEndSubscription: Subscription;
|
||||
|
||||
constructor(
|
||||
private clipboardService: ClipboardService,
|
||||
public dialog: MatDialog,
|
||||
public electronService: ElectronService,
|
||||
private wsConnectionService: WsConnectionService,
|
||||
private appService: AppService,
|
||||
private _snackBar: MatSnackBar,
|
||||
private ngZone: NgZone
|
||||
) {
|
||||
this.websocketHandlerPromise = this.wsConnectionService.getConnection();
|
||||
}
|
||||
|
||||
websocketHandlerPromise: Promise<WsHandler>;
|
||||
) {}
|
||||
|
||||
ngOnInit() {
|
||||
this.clipboardService.links.subscribe(e => {
|
||||
@@ -48,10 +35,6 @@ export class HomeComponent implements OnInit, OnDestroy {
|
||||
});
|
||||
}
|
||||
|
||||
view(chip) {
|
||||
console.log(chip);
|
||||
}
|
||||
|
||||
ngOnDestroy() {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { LoginComponent } from './login.component';
|
||||
|
||||
describe('LoginComponent', () => {
|
||||
let component: LoginComponent;
|
||||
let fixture: ComponentFixture<LoginComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ LoginComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(LoginComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,6 @@
|
||||
<ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-material" style="width: 100%; height: 100%;">
|
||||
</ag-grid-angular>
|
||||
<div style="height: 4px; width: 100%; padding: 5px 0 0">
|
||||
<mat-progress-bar *ngIf="loading" mode="indeterminate"></mat-progress-bar>
|
||||
<mat-progress-bar *ngIf="loading | async" mode="indeterminate"></mat-progress-bar>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { MultiPostComponent } from './multi-post.component';
|
||||
|
||||
describe('MultiPostComponent', () => {
|
||||
let component: MultiPostComponent;
|
||||
let fixture: ComponentFixture<MultiPostComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ MultiPostComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(MultiPostComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { VRPostParse, VRThreadParseState } from './../common/vr-post-parse.model';
|
||||
import { Component, OnInit, NgZone, OnDestroy, Input, Output, EventEmitter } from '@angular/core';
|
||||
import { Component, OnInit, NgZone, OnDestroy, Input, Output, EventEmitter, ChangeDetectionStrategy, AfterViewInit } from '@angular/core';
|
||||
import { MatSnackBar } from '@angular/material';
|
||||
import { GridOptions } from 'ag-grid-community';
|
||||
import { UrlRendererComponent } from './url-renderer.component';
|
||||
@@ -14,9 +14,10 @@ import { ServerService } from '../server-service';
|
||||
@Component({
|
||||
selector: 'app-multi-post',
|
||||
templateUrl: './multi-post.component.html',
|
||||
styleUrls: ['./multi-post.component.scss']
|
||||
styleUrls: ['./multi-post.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class MultiPostComponent implements OnInit, OnDestroy {
|
||||
export class MultiPostComponent implements OnInit, OnDestroy, AfterViewInit {
|
||||
|
||||
@Input()
|
||||
threadId: string;
|
||||
@@ -27,7 +28,7 @@ export class MultiPostComponent implements OnInit, OnDestroy {
|
||||
gridOptions: GridOptions;
|
||||
websocketHandlerPromise: Promise<WsHandler>;
|
||||
subscription: Subscription;
|
||||
loading = true;
|
||||
loading: EventEmitter<boolean> = new EventEmitter();
|
||||
|
||||
constructor(
|
||||
private ngZone: NgZone,
|
||||
@@ -39,6 +40,10 @@ export class MultiPostComponent implements OnInit, OnDestroy {
|
||||
this.websocketHandlerPromise = this.wsConnectionService.getConnection();
|
||||
}
|
||||
|
||||
ngAfterViewInit(): void {
|
||||
this.loading.emit(true);
|
||||
}
|
||||
|
||||
ngOnInit(): void {
|
||||
this.gridOptions = <GridOptions>{
|
||||
columnDefs: [
|
||||
@@ -108,18 +113,27 @@ export class MultiPostComponent implements OnInit, OnDestroy {
|
||||
connect() {
|
||||
this.websocketHandlerPromise.then((handler: WsHandler) => {
|
||||
console.log('Connecting to thread parsing');
|
||||
let count = 0;
|
||||
let data_0;
|
||||
this.subscription = handler.subscribeForThreadParsing(
|
||||
(states: Array<VRThreadParseState>, data: Array<VRPostParse>) => {
|
||||
if (data[0].threadId === this.threadId) {
|
||||
if (data.length > 0 && data[0].threadId === this.threadId) {
|
||||
this.gridOptions.api.updateRowData({ add: data });
|
||||
count += data.length;
|
||||
data_0 = data[0];
|
||||
}
|
||||
|
||||
if (states.length > 0 && states[states.length - 1].state === 'END' && states[0].threadId === this.threadId) {
|
||||
this.ngZone.run(() => {
|
||||
this.loading = false;
|
||||
if (this.gridOptions.api.getDisplayedRowCount() === 1) {
|
||||
this.loading.emit(false);
|
||||
if (count === 0) {
|
||||
this._snackBar.open('No posts were found', null, {
|
||||
duration: 5000
|
||||
});
|
||||
this.done.emit(true);
|
||||
} else if (count === 1) {
|
||||
this.addPosts(
|
||||
[this.gridOptions.api.getDisplayedRowAtIndex(0).data].map(e => ({
|
||||
[data_0].map(e => ({
|
||||
threadId: e.threadId,
|
||||
postId: e.postId
|
||||
}))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Component, OnInit, OnDestroy } from '@angular/core';
|
||||
import { Component, OnInit, OnDestroy, ChangeDetectionStrategy } from '@angular/core';
|
||||
import { AgRendererComponent } from 'ag-grid-angular';
|
||||
import { ICellRendererParams } from 'ag-grid-community';
|
||||
import { VRPostParse } from '../common/vr-post-parse.model';
|
||||
@@ -10,7 +10,8 @@ import { ElectronService } from 'ngx-electron';
|
||||
<a href="javascript:void(0)" [appPreview]="postResult.previews" (click)="goTo()"
|
||||
>https://vipergirls/threads/?p={{ postResult.postId }}</a
|
||||
>
|
||||
`
|
||||
`,
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class UrlRendererComponent implements OnInit, OnDestroy, AgRendererComponent {
|
||||
constructor(public electronService: ElectronService) {}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { PostDetailComponent } from './post-detail.component';
|
||||
|
||||
describe('PostDetailComponent', () => {
|
||||
let component: PostDetailComponent;
|
||||
let fixture: ComponentFixture<PostDetailComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ PostDetailComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(PostDetailComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { PostDetailsProgressRendererComponent } from './post-details-progress.component';
|
||||
import { WsConnectionService } from './../ws-connection.service';
|
||||
import { Component, OnInit, Inject, OnDestroy, NgZone } from '@angular/core';
|
||||
import { Component, OnInit, Inject, OnDestroy, NgZone, ChangeDetectionStrategy } from '@angular/core';
|
||||
import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material';
|
||||
import { PostDetailsDataSource } from './post-details.datasource';
|
||||
import { PostState } from '../posts/post-state.model';
|
||||
@@ -9,7 +9,8 @@ import { GridOptions } from 'ag-grid-community';
|
||||
@Component({
|
||||
selector: 'app-post-detail',
|
||||
templateUrl: './post-detail.component.html',
|
||||
styleUrls: ['./post-detail.component.scss']
|
||||
styleUrls: ['./post-detail.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class PostDetailComponent implements OnInit, OnDestroy {
|
||||
|
||||
|
||||
@@ -1,27 +1,32 @@
|
||||
<div class="container" style="height: 100%; background-color: white">
|
||||
<div
|
||||
class="progress-bar-back"
|
||||
style="position: relative; height: 100%;"
|
||||
>
|
||||
<ng-container *ngIf="postDetails$ | async as details">
|
||||
<div
|
||||
class="progress-foreground"
|
||||
fxLayout="row"
|
||||
fxLayoutAlign="space-between"
|
||||
style="background-color: transparent; width: 100%; height: 100%;"
|
||||
[ngClass]="{
|
||||
error: details.status === 'ERROR'
|
||||
}"
|
||||
class="progress-bar-back"
|
||||
style="position: relative; height: 100%;"
|
||||
>
|
||||
<div class="progress-bar" style="position: absolute; width: 100%; top: 37px; padding: 0 20px">
|
||||
<mat-progress-bar [value]="postDetails.progress" class="example-margin" color="primary" mode="determinate">
|
||||
</mat-progress-bar>
|
||||
<div
|
||||
class="progress-foreground"
|
||||
fxLayout="row"
|
||||
fxLayoutAlign="space-between"
|
||||
style="background-color: transparent; width: 100%; height: 100%;"
|
||||
>
|
||||
<div class="progress-bar" style="position: absolute; width: 100%; top: 37px; padding: 0 20px">
|
||||
<mat-progress-bar [value]="details.progress" class="example-margin" color="primary" mode="determinate">
|
||||
</mat-progress-bar>
|
||||
</div>
|
||||
<span fxLayout="row">
|
||||
<span style="padding-left: 20px"
|
||||
><a (click)="goTo()" href="javascript:void(0)" style="color: rgba(0, 0, 0, 0.87);">{{
|
||||
details.url
|
||||
}}</a></span
|
||||
>
|
||||
<span class="filler" fxFlex="grow"></span>
|
||||
</span>
|
||||
<span class="progress-percentage" style="padding-right: 20px">{{ trunc(details.progress) + '%' }}</span>
|
||||
</div>
|
||||
<span fxLayout="row">
|
||||
<span style="padding-left: 20px"
|
||||
><a (click)="goTo()" href="javascript:void(0)" style="color: rgba(0, 0, 0, 0.87);">{{
|
||||
postDetails.url
|
||||
}}</a></span
|
||||
>
|
||||
<span class="filler" fxFlex="grow"></span>
|
||||
</span>
|
||||
<span class="progress-percentage" style="padding-right: 20px">{{ trunc(postDetails.progress) + '%' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
</div>
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
@mixin post-detail-renderer-component-theme($theme) {
|
||||
$blue: mat-palette($mat-blue);
|
||||
$red: mat-palette($mat-red);
|
||||
|
||||
.progress-bar .mat-progress-bar-fill::after {
|
||||
background-color: mat-color($blue, 300);
|
||||
@@ -10,4 +11,14 @@
|
||||
.progress-bar .mat-progress-bar-buffer {
|
||||
background-color: mat-color($blue, 100);
|
||||
}
|
||||
|
||||
app-details-cell .error {
|
||||
background-color: mat-color($red, 50);
|
||||
& .progress-bar .mat-progress-bar-fill::after {
|
||||
background-color: mat-color($red, 300);
|
||||
}
|
||||
& .progress-bar .mat-progress-bar-buffer {
|
||||
background-color: mat-color($red, 100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
import { WsConnectionService } from '../ws-connection.service';
|
||||
import { Component, OnInit, OnDestroy, NgZone } from '@angular/core';
|
||||
import {
|
||||
Component,
|
||||
OnInit,
|
||||
OnDestroy,
|
||||
NgZone,
|
||||
ChangeDetectionStrategy,
|
||||
EventEmitter,
|
||||
AfterViewInit
|
||||
} from '@angular/core';
|
||||
import { AgRendererComponent } from 'ag-grid-angular';
|
||||
import { Subscription } from 'rxjs';
|
||||
import { PostDetails } from './post-details.model';
|
||||
@@ -10,9 +18,10 @@ import { ElectronService } from 'ngx-electron';
|
||||
@Component({
|
||||
selector: 'app-details-cell',
|
||||
templateUrl: 'post-details-progress.component.html',
|
||||
styleUrls: ['post-details-progress.component.scss']
|
||||
styleUrls: ['post-details-progress.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class PostDetailsProgressRendererComponent implements AgRendererComponent, OnInit, OnDestroy {
|
||||
export class PostDetailsProgressRendererComponent implements AgRendererComponent, OnInit, OnDestroy, AfterViewInit {
|
||||
constructor(
|
||||
private wsConnectionService: WsConnectionService,
|
||||
private zone: NgZone,
|
||||
@@ -23,8 +32,8 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
|
||||
|
||||
websocketHandlerPromise: Promise<WsHandler>;
|
||||
subscription: Subscription;
|
||||
params: ICellRendererParams;
|
||||
postDetails: PostDetails;
|
||||
postDetails$: EventEmitter<PostDetails> = new EventEmitter();
|
||||
private postDetails: PostDetails;
|
||||
|
||||
trunc(value: number): number {
|
||||
return Math.trunc(value);
|
||||
@@ -37,6 +46,7 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
|
||||
e.forEach(v => {
|
||||
if (this.postDetails.url === v.url) {
|
||||
this.postDetails = v;
|
||||
this.postDetails$.emit(this.postDetails);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -52,6 +62,10 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
|
||||
}
|
||||
}
|
||||
|
||||
ngAfterViewInit(): void {
|
||||
this.postDetails$.emit(this.postDetails);
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
if (this.subscription != null) {
|
||||
this.subscription.unsubscribe();
|
||||
@@ -59,11 +73,12 @@ export class PostDetailsProgressRendererComponent implements AgRendererComponent
|
||||
}
|
||||
|
||||
agInit(params: ICellRendererParams): void {
|
||||
this.params = params;
|
||||
this.postDetails = params.data;
|
||||
}
|
||||
|
||||
refresh(params: ICellRendererParams): boolean {
|
||||
return false;
|
||||
this.postDetails = params.data;
|
||||
this.postDetails$.emit(this.postDetails);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ export class PostDetailsDataSource {
|
||||
private postId: string,
|
||||
private zone: NgZone
|
||||
) {
|
||||
this.websocketHandlerPromise = wsConnectionService.getConnection();
|
||||
this.websocketHandlerPromise = this.wsConnectionService.getConnection();
|
||||
}
|
||||
|
||||
websocketHandlerPromise: Promise<WsHandler>;
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
import { GridApi } from 'ag-grid-community';
|
||||
|
||||
@Injectable()
|
||||
export class PostsDataService {
|
||||
private api: GridApi;
|
||||
|
||||
public setGridApi(api: GridApi) {
|
||||
this.api = api;
|
||||
}
|
||||
|
||||
public remove(toRemove: { postId: string }[]) {
|
||||
const removeTx = [];
|
||||
toRemove.forEach(element => {
|
||||
const nodeToDelete = this.api.getRowNode(element.postId);
|
||||
if (nodeToDelete != null) {
|
||||
removeTx.push(nodeToDelete.data);
|
||||
}
|
||||
});
|
||||
this.api.updateRowData({ remove: removeTx });
|
||||
}
|
||||
|
||||
search(event) {
|
||||
this.api.setQuickFilter(event);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
<div class="container" style="height: 100%; background-color: white">
|
||||
<!-- <div class="container" style="height: 100%; background-color: white">
|
||||
<div
|
||||
[ngClass]="{
|
||||
'error-back': postState.status === 'ERROR',
|
||||
'downloading-back': postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL',
|
||||
'complete-back': postState.status === 'COMPLETE'
|
||||
error: postState.status === 'ERROR',
|
||||
downloading: postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL',
|
||||
complete: postState.status === 'COMPLETE',
|
||||
stopped: postState.status === 'STOPPED'
|
||||
}"
|
||||
class="progress-bar-back"
|
||||
style="position: relative; height: 100%; max-height: 48px;"
|
||||
@@ -41,7 +42,6 @@
|
||||
<button
|
||||
(click)="restart()"
|
||||
*ngIf="
|
||||
postState.status === 'PENDING' ||
|
||||
(postState.status === 'COMPLETE' && postState.progress !== 100) ||
|
||||
postState.status === 'ERROR' ||
|
||||
postState.status === 'STOPPED'
|
||||
@@ -53,7 +53,9 @@
|
||||
</button>
|
||||
<button
|
||||
(click)="stop()"
|
||||
*ngIf="postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL'"
|
||||
*ngIf="
|
||||
postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL' || postState.status === 'PENDING'
|
||||
"
|
||||
mat-menu-item
|
||||
>
|
||||
<ng-container>
|
||||
@@ -89,7 +91,7 @@
|
||||
<h4 class="cell attribute">Post URL:</h4>
|
||||
<p class="cell value">
|
||||
<a (click)="goTo()" [appPreview]="postState.previews" href="javascript:void(0)"
|
||||
>https://vipergirls/threads/?p={{ postState.postId }}</a
|
||||
>https://vipergirls/threads/?p={{ postState.postId }}</a
|
||||
>
|
||||
</p>
|
||||
</div>
|
||||
@@ -101,4 +103,53 @@
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div> -->
|
||||
|
||||
<div class="container" style="height: 100%; background-color: white">
|
||||
<ng-container *ngIf="postState$ | async as postState">
|
||||
<div
|
||||
[ngClass]="{
|
||||
error: postState.status === 'ERROR',
|
||||
downloading: postState.status === 'DOWNLOADING' || postState.status === 'PARTIAL',
|
||||
complete: postState.status === 'COMPLETE',
|
||||
stopped: postState.status === 'STOPPED'
|
||||
}"
|
||||
class="progress-bar-back"
|
||||
style="position: relative; height: 100%; transition: background-color 0.5s;"
|
||||
>
|
||||
<div
|
||||
class="progress-foreground"
|
||||
fxLayout="row"
|
||||
fxLayoutAlign="start"
|
||||
fxLayoutGap="5px"
|
||||
style="background-color: transparent; width: 100%; height: 100%;"
|
||||
>
|
||||
<div class="checkbox" style="display: inline-block; width: 5px; height: 100%;"></div>
|
||||
<span fxFlex="grow" fxLayout="row" fxLayoutAlign="space-between center">
|
||||
<span
|
||||
><a (click)="goTo()" [appPreview]="postState.previews" href="javascript:void(0)">{{
|
||||
postState.title
|
||||
}}</a></span
|
||||
>
|
||||
<span>{{postState.done + '/' + postState.total}} done from {{postState.hosts}}</span>
|
||||
</span>
|
||||
<button [matMenuTriggerFor]="menu" mat-icon-button>
|
||||
<mat-icon>more_vert</mat-icon>
|
||||
</button>
|
||||
<mat-menu #menu="matMenu">
|
||||
<button (click)="seeDetails()" mat-menu-item>
|
||||
<mat-icon>list</mat-icon>
|
||||
<span>Files</span>
|
||||
</button>
|
||||
<button (click)="open()" *ngIf="electronService.isElectronApp" mat-menu-item>
|
||||
<mat-icon>open_in_new</mat-icon>
|
||||
<span>Download Location</span>
|
||||
</button>
|
||||
</mat-menu>
|
||||
</div>
|
||||
<div class="progress-bar" color="accent" style="position: absolute; width: 100%; top: 37px; padding: 0 10px">
|
||||
<mat-progress-bar [value]="postState.progress" color="primary" mode="determinate"></mat-progress-bar>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
</div>
|
||||
|
||||
@@ -1,28 +1,3 @@
|
||||
.progress-bar-back {
|
||||
transition: background-color 0.5s;
|
||||
}
|
||||
|
||||
.table {
|
||||
display: table;
|
||||
}
|
||||
.row {
|
||||
display: table-row;
|
||||
}
|
||||
.cell {
|
||||
display: table-cell;
|
||||
}
|
||||
|
||||
section {
|
||||
width: 100%;
|
||||
position: absolute;
|
||||
top: 48px;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.details {
|
||||
line-height: 24px;
|
||||
}
|
||||
|
||||
.value {
|
||||
padding-left: 5px;
|
||||
.checkbox {
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
@@ -6,14 +6,14 @@
|
||||
$green: mat-palette($mat-green);
|
||||
|
||||
app-progress-cell .progress-bar .mat-progress-bar-fill::after {
|
||||
background-color: mat-color($grey, 300);
|
||||
background-color: mat-color($grey, 400);
|
||||
}
|
||||
|
||||
app-progress-cell .progress-bar .mat-progress-bar-buffer {
|
||||
background-color: mat-color($grey, 100);
|
||||
background-color: mat-color($grey, 200);
|
||||
}
|
||||
|
||||
app-progress-cell .error-back {
|
||||
app-progress-cell .error {
|
||||
background-color: mat-color($red, 50);
|
||||
& .progress-bar .mat-progress-bar-fill::after {
|
||||
background-color: mat-color($red, 300);
|
||||
@@ -23,7 +23,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
app-progress-cell .downloading-back {
|
||||
app-progress-cell .downloading {
|
||||
background-color: mat-color($green, 50);
|
||||
& .progress-bar .mat-progress-bar-fill::after {
|
||||
background-color: mat-color($green, 300);
|
||||
@@ -33,7 +33,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
app-progress-cell .complete-back {
|
||||
app-progress-cell .complete {
|
||||
background-color: mat-color($green, 100);
|
||||
& .progress-bar .mat-progress-bar-fill::after {
|
||||
background-color: mat-color($green, 300);
|
||||
@@ -42,4 +42,8 @@
|
||||
background-color: mat-color($green, 100);
|
||||
}
|
||||
}
|
||||
|
||||
app-progress-cell .stopped {
|
||||
background-color: mat-color($grey, 100);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import { SharedService } from './shared.service';
|
||||
import { WsConnectionService } from '../ws-connection.service';
|
||||
import { PostState } from './post-state.model';
|
||||
import { Component, OnInit, OnDestroy, NgZone } from '@angular/core';
|
||||
import {
|
||||
Component,
|
||||
OnInit,
|
||||
OnDestroy,
|
||||
NgZone,
|
||||
ChangeDetectionStrategy,
|
||||
AfterViewInit,
|
||||
EventEmitter
|
||||
} from '@angular/core';
|
||||
import { AgRendererComponent } from 'ag-grid-angular';
|
||||
import { Subscription, Observable } from 'rxjs';
|
||||
import { WsHandler } from '../ws-handler';
|
||||
@@ -12,27 +19,24 @@ import { MatDialog, MatSnackBar } from '@angular/material';
|
||||
import { PostDetailComponent } from '../post-detail/post-detail.component';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { ServerService } from '../server-service';
|
||||
import { ConfirmDialogComponent } from '../common/confirmation-component/confirmation-dialog';
|
||||
import { filter, flatMap } from 'rxjs/operators';
|
||||
import { RemoveResponse } from '../common/remove-response.model';
|
||||
import { DownloadPath } from '../common/download-path.model';
|
||||
|
||||
@Component({
|
||||
selector: 'app-progress-cell',
|
||||
templateUrl: 'post-progress.renderer.component.html',
|
||||
styleUrls: ['post-progress.renderer.component.scss']
|
||||
styleUrls: ['post-progress.renderer.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class PostProgressRendererComponent implements AgRendererComponent, OnInit, OnDestroy {
|
||||
export class PostProgressRendererComponent implements AgRendererComponent, OnInit, OnDestroy, AfterViewInit {
|
||||
constructor(
|
||||
private wsConnectionService: WsConnectionService,
|
||||
private zone: NgZone,
|
||||
private sharedService: SharedService,
|
||||
public electronService: ElectronService,
|
||||
private breakpointObserver: BreakpointObserver,
|
||||
public dialog: MatDialog,
|
||||
private httpClient: HttpClient,
|
||||
private serverService: ServerService,
|
||||
private _snackBar: MatSnackBar,
|
||||
private _snackBar: MatSnackBar
|
||||
) {
|
||||
this.websocketHandlerPromise = this.wsConnectionService.getConnection();
|
||||
if (this.electronService.isElectronApp) {
|
||||
@@ -41,10 +45,9 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
|
||||
}
|
||||
|
||||
websocketHandlerPromise: Promise<WsHandler>;
|
||||
params: ICellRendererParams;
|
||||
postState: PostState;
|
||||
postState$: EventEmitter<PostState> = new EventEmitter();
|
||||
private postState: PostState;
|
||||
updatesSubscription: Subscription;
|
||||
expandSubscription: Subscription;
|
||||
expanded = false;
|
||||
isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
|
||||
fs;
|
||||
@@ -60,47 +63,32 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
|
||||
e.forEach(v => {
|
||||
if (this.postState.postId === v.postId) {
|
||||
this.postState = v;
|
||||
this.postState$.emit(this.postState);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
this.expandSubscription = this.sharedService.expandedPost.subscribe(postId => {
|
||||
if (this.expanded && this.postState.postId !== postId) {
|
||||
this.toggleExpand();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
ngAfterViewInit(): void {
|
||||
this.postState$.emit(this.postState);
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
if (this.updatesSubscription != null) {
|
||||
this.updatesSubscription.unsubscribe();
|
||||
|
||||
}
|
||||
if (this.expandSubscription != null) {
|
||||
this.expandSubscription.unsubscribe();
|
||||
}
|
||||
}
|
||||
|
||||
agInit(params: ICellRendererParams): void {
|
||||
this.params = params;
|
||||
this.postState = params.data;
|
||||
this.params.node.setRowHeight(48);
|
||||
}
|
||||
|
||||
refresh(params: ICellRendererParams): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
toggleExpand() {
|
||||
if (this.expanded) {
|
||||
this.params.node.setRowHeight(48);
|
||||
} else {
|
||||
this.params.node.setRowHeight(130);
|
||||
this.sharedService.publishExpanded(this.postState.postId);
|
||||
}
|
||||
this.params.api.onRowHeightChanged();
|
||||
this.expanded = !this.expanded;
|
||||
this.postState = params.data;
|
||||
this.postState$.emit(this.postState);
|
||||
return true;
|
||||
}
|
||||
|
||||
goTo() {
|
||||
@@ -135,52 +123,6 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
|
||||
});
|
||||
}
|
||||
|
||||
restart() {
|
||||
this.httpClient.post(this.serverService.baseUrl + '/post/restart', { postId: this.postState.postId }).subscribe(
|
||||
() => {
|
||||
this._snackBar.open('Download started', null, {
|
||||
duration: 5000
|
||||
});
|
||||
},
|
||||
error => {
|
||||
console.error(error);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
remove() {
|
||||
this.dialog
|
||||
.open(ConfirmDialogComponent, {
|
||||
maxHeight: '100vh',
|
||||
maxWidth: '100vw',
|
||||
height: '200px',
|
||||
width: '60%',
|
||||
data: { header: 'Confirmation', content: 'Are you sure you want to remove this item ?' }
|
||||
})
|
||||
.afterClosed()
|
||||
.pipe(
|
||||
filter(e => e === 'yes'),
|
||||
flatMap(e =>
|
||||
this.httpClient.post<RemoveResponse>(this.serverService.baseUrl + '/post/remove', {
|
||||
postId: this.postState.postId
|
||||
})
|
||||
)
|
||||
)
|
||||
.subscribe(
|
||||
data => {
|
||||
const toRemove = [];
|
||||
const nodeToDelete = this.params.api.getRowNode(data.postId);
|
||||
if (nodeToDelete != null) {
|
||||
toRemove.push(nodeToDelete.data);
|
||||
}
|
||||
this.params.api.updateRowData({ remove: toRemove });
|
||||
},
|
||||
error => {
|
||||
console.error(error);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
open() {
|
||||
if (!this.electronService.isElectronApp) {
|
||||
console.error('Cannot open downloader folder, not electron app');
|
||||
@@ -192,7 +134,7 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
|
||||
if (this.fs.existsSync(path.path)) {
|
||||
this.electronService.shell.openItem(path.path);
|
||||
} else {
|
||||
if(this.postState.done <= 0) {
|
||||
if (this.postState.done <= 0) {
|
||||
this._snackBar.open('Download has not been started yet for this post', null, {
|
||||
duration: 5000
|
||||
});
|
||||
@@ -208,17 +150,4 @@ export class PostProgressRendererComponent implements AgRendererComponent, OnIni
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.httpClient.post(this.serverService.baseUrl + '/post/stop', { postId: this.postState.postId }).subscribe(
|
||||
() => {
|
||||
this._snackBar.open('Download stopped', null, {
|
||||
duration: 5000
|
||||
});
|
||||
},
|
||||
error => {
|
||||
console.error(error);
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,2 @@
|
||||
<div fxLayout="row" fxLayoutAlign="center center">
|
||||
<!-- <app-filter fxFlex="noshrink"></app-filter> -->
|
||||
<span fxFlex="grow"></span>
|
||||
<form autocomplete="off">
|
||||
<mat-form-field>
|
||||
<input (ngModelChange)="search($event)" matInput name="search" ngModel placeholder="Search"/>
|
||||
<mat-icon matSuffix>search</mat-icon>
|
||||
</mat-form-field>
|
||||
</form>
|
||||
</div>
|
||||
<ag-grid-angular style="width: 100%; height: 100%;" class="ag-theme-material" [gridOptions]="gridOptions">
|
||||
</ag-grid-angular>
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
$menu-width: 60px;
|
||||
$cell-space: 8px;
|
||||
|
||||
.title-cell {
|
||||
flex: 2 0 calc(60% - $menu-width);
|
||||
margin-right: $cell-space;
|
||||
}
|
||||
|
||||
.progress-cell {
|
||||
flex: 1 0 calc(30% - $menu-width);
|
||||
}
|
||||
|
||||
.menu-cell {
|
||||
flex: 0 0 $menu-width;
|
||||
justify-content: flex-end;
|
||||
margin-left: $cell-space;
|
||||
}
|
||||
|
||||
:host ::ng-deep .ag-row-hover {
|
||||
background-color: transparent !important;
|
||||
}
|
||||
@@ -1,17 +1,24 @@
|
||||
import { Component, OnInit, OnDestroy, NgZone } from '@angular/core';
|
||||
import { PostsDataService } from './../posts-data.service';
|
||||
import { SelectionService } from './../selection-service';
|
||||
import { Component, OnInit, OnDestroy, NgZone, ChangeDetectionStrategy, AfterViewInit } from '@angular/core';
|
||||
import { PostsDataSource } from './post.datasource';
|
||||
import { WsConnectionService } from '../ws-connection.service';
|
||||
import { GridOptions, IFilterComp } from 'ag-grid-community';
|
||||
import { GridOptions } from 'ag-grid-community';
|
||||
import { PostProgressRendererComponent } from './post-progress.renderer.component';
|
||||
import { Subject } from 'rxjs';
|
||||
|
||||
@Component({
|
||||
selector: 'app-posts',
|
||||
templateUrl: './posts.component.html',
|
||||
styleUrls: ['./posts.component.scss']
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class PostsComponent implements OnInit, OnDestroy {
|
||||
constructor(private wsConnection: WsConnectionService, private zone: NgZone) {
|
||||
export class PostsComponent implements OnInit, OnDestroy, AfterViewInit {
|
||||
constructor(
|
||||
private wsConnection: WsConnectionService,
|
||||
private zone: NgZone,
|
||||
private selectionService: SelectionService,
|
||||
private postsDataService: PostsDataService
|
||||
) {
|
||||
this.gridOptions = <GridOptions>{
|
||||
columnDefs: [
|
||||
{
|
||||
@@ -20,11 +27,15 @@ export class PostsComponent implements OnInit, OnDestroy {
|
||||
sortable: true,
|
||||
cellRenderer: 'progressCellRenderer',
|
||||
cellClass: 'no-padding',
|
||||
sort: 'asc'
|
||||
sort: 'asc',
|
||||
headerCheckboxSelection: true,
|
||||
headerCheckboxSelectionFilteredOnly: true
|
||||
}
|
||||
],
|
||||
rowHeight: 48,
|
||||
animateRows: true,
|
||||
rowSelection: 'multiple',
|
||||
rowDeselection: true,
|
||||
rowData: [],
|
||||
frameworkComponents: {
|
||||
progressCellRenderer: PostProgressRendererComponent
|
||||
@@ -38,7 +49,8 @@ export class PostsComponent implements OnInit, OnDestroy {
|
||||
this.dataSource.connect();
|
||||
},
|
||||
onGridSizeChanged: () => this.gridOptions.api.sizeColumnsToFit(),
|
||||
onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit()
|
||||
onRowDataUpdated: () => this.gridOptions.api.sizeColumnsToFit(),
|
||||
onSelectionChanged: () => this.selectionService.onSelectionChanged(this.gridOptions.api.getSelectedNodes())
|
||||
};
|
||||
}
|
||||
|
||||
@@ -46,28 +58,12 @@ export class PostsComponent implements OnInit, OnDestroy {
|
||||
gridOptions: GridOptions;
|
||||
dataSource: PostsDataSource;
|
||||
|
||||
search(event) {
|
||||
this.gridOptions.api.setQuickFilter(event);
|
||||
}
|
||||
|
||||
removeRows(postIds: string[]): void {
|
||||
if (postIds == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const toRemove = [];
|
||||
|
||||
postIds.forEach(p => {
|
||||
const nodeToDelete = this.gridOptions.api.getRowNode(p);
|
||||
if (nodeToDelete != null) {
|
||||
toRemove.push(nodeToDelete.data);
|
||||
}
|
||||
});
|
||||
this.gridOptions.api.updateRowData({ remove: toRemove });
|
||||
}
|
||||
|
||||
ngOnInit() {}
|
||||
|
||||
ngAfterViewInit(): void {
|
||||
this.postsDataService.setGridApi(this.gridOptions.api);
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
this.dataSource.disconnect();
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<h2 class="no-wrap" mat-dialog-title>Scan</h2>
|
||||
</div>
|
||||
<mat-dialog-content fxFlex="grow" fxLayout="column">
|
||||
<div *ngIf="!hideScan">
|
||||
<div *ngIf="!(hideScan | async)">
|
||||
<form
|
||||
#f="ngForm"
|
||||
(ngSubmit)="submit(f)"
|
||||
@@ -13,21 +13,16 @@
|
||||
fxLayoutGap="20px"
|
||||
>
|
||||
<mat-form-field fxFlex="grow">
|
||||
<input
|
||||
[(ngModel)]="input"
|
||||
matInput
|
||||
name="url"
|
||||
placeholder="Put a vipergirls.to link"
|
||||
required
|
||||
/>
|
||||
<input [(ngModel)]="input" matInput name="url" placeholder="Put a vipergirls.to link" required/>
|
||||
</mat-form-field>
|
||||
<div>
|
||||
<button [disabled]="f.invalid" color="primary" mat-raised-button type="submit">Scan</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<ng-container *ngIf="threadId != null">
|
||||
<app-multi-post (done)="done($event)" [threadId]="threadId" style="height: 100%"></app-multi-post>
|
||||
<ng-container *ngIf="threadId | async as thId">
|
||||
<app-multi-post (done)="done($event)" *ngIf="thId != null" [threadId]="thId"
|
||||
style="height: 100%"></app-multi-post>
|
||||
</ng-container>
|
||||
</mat-dialog-content>
|
||||
<mat-dialog-actions align="end">
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { ScanComponent } from './scan.component';
|
||||
|
||||
describe('ScanComponent', () => {
|
||||
let component: ScanComponent;
|
||||
let fixture: ComponentFixture<ScanComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ ScanComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(ScanComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,13 @@
|
||||
import { Component, OnInit, NgZone, ViewChild, Inject } from '@angular/core';
|
||||
import {
|
||||
Component,
|
||||
OnInit,
|
||||
NgZone,
|
||||
ViewChild,
|
||||
Inject,
|
||||
ChangeDetectionStrategy,
|
||||
EventEmitter,
|
||||
AfterViewInit
|
||||
} from '@angular/core';
|
||||
import { NgForm } from '@angular/forms';
|
||||
import { MatSnackBar, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
@@ -9,9 +18,10 @@ import { MultiPostComponent } from '../multi-post/multi-post.component';
|
||||
@Component({
|
||||
selector: 'app-scan',
|
||||
templateUrl: './scan.component.html',
|
||||
styleUrls: ['./scan.component.scss']
|
||||
styleUrls: ['./scan.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class ScanComponent implements OnInit {
|
||||
export class ScanComponent implements OnInit, AfterViewInit {
|
||||
constructor(
|
||||
private ngZone: NgZone,
|
||||
private httpClient: HttpClient,
|
||||
@@ -25,13 +35,11 @@ export class ScanComponent implements OnInit {
|
||||
multipost: MultiPostComponent;
|
||||
|
||||
input: string;
|
||||
threadId: string;
|
||||
hideScan = false;
|
||||
threadId: EventEmitter<string> = new EventEmitter();
|
||||
hideScan: EventEmitter<boolean> = new EventEmitter();
|
||||
|
||||
submit(form: NgForm) {
|
||||
this.ngZone.run(() => {
|
||||
this.hideScan = true;
|
||||
this.threadId = null;
|
||||
this.processUrl(this.input, form);
|
||||
});
|
||||
}
|
||||
@@ -45,6 +53,8 @@ export class ScanComponent implements OnInit {
|
||||
}
|
||||
|
||||
processUrl(url: string, form?: NgForm) {
|
||||
this.hideScan.emit(true);
|
||||
this.threadId.emit(null);
|
||||
this.httpClient
|
||||
.post<{ threadId: string; postId: string }>(this.serverService.baseUrl + '/post', { url: url })
|
||||
.pipe(
|
||||
@@ -77,7 +87,13 @@ export class ScanComponent implements OnInit {
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.threadId = response.threadId;
|
||||
this.threadId.emit(response.threadId);
|
||||
});
|
||||
},
|
||||
error => {
|
||||
this.hideScan.emit(false);
|
||||
this._snackBar.open(error.error, null, {
|
||||
duration: 5000
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -91,11 +107,15 @@ export class ScanComponent implements OnInit {
|
||||
});
|
||||
}
|
||||
|
||||
ngOnInit() {
|
||||
ngOnInit() {}
|
||||
|
||||
ngAfterViewInit(): void {
|
||||
this.ngZone.run(() => {
|
||||
if (this.data.url != null) {
|
||||
this.hideScan = true;
|
||||
this.hideScan.emit(true);
|
||||
this.processUrl(this.data.url);
|
||||
} else {
|
||||
this.hideScan.emit(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
import { Subject, Observable } from 'rxjs';
|
||||
import { RowNode } from 'ag-grid-community';
|
||||
|
||||
@Injectable()
|
||||
export class SelectionService {
|
||||
|
||||
private _selected$: Subject<RowNode[]> = new Subject();
|
||||
|
||||
get selected$(): Observable<RowNode[]> {
|
||||
return this._selected$.asObservable();
|
||||
}
|
||||
|
||||
public onSelectionChanged(selected: RowNode[]) {
|
||||
this._selected$.next(selected);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
<div fxLayout="column" class="dialog-container" style="height: 100%;">
|
||||
<div fxFlex="nogrow" fxLayout="row" fxLayoutAlign="space-between center">
|
||||
<h2 class="no-wrap" mat-dialog-title>Settings</h2>
|
||||
<mat-slide-toggle (change)="updateTheme()" [(ngModel)]="darkTheme">{{darkTheme ? 'Dark theme' : 'Light theme'}}
|
||||
<mat-slide-toggle (change)="updateTheme()" [(ngModel)]="darkTheme"
|
||||
>{{ darkTheme ? 'Dark theme' : 'Light theme' }}
|
||||
</mat-slide-toggle>
|
||||
</div>
|
||||
<mat-dialog-content fxFlex="grow">
|
||||
@@ -32,20 +33,32 @@
|
||||
placeholder="Max concurrent downloads"
|
||||
name="maxThreads"
|
||||
min="1"
|
||||
max="8"
|
||||
max="4"
|
||||
required
|
||||
/>
|
||||
</mat-form-field>
|
||||
|
||||
<mat-checkbox color="primary" formControlName="subLocation" name="subLocation"
|
||||
>Save posts in sub folders
|
||||
</mat-checkbox>
|
||||
|
||||
<mat-checkbox *ngIf="generalSettingsForm.get('subLocation').value" color="primary"
|
||||
formControlName="threadSubLocation" name="threadSubLocation" style="margin-left: 15px"
|
||||
>Create a subfolder per thread
|
||||
</mat-checkbox>
|
||||
|
||||
<mat-checkbox color="primary" formControlName="forceOrder" name="forceOrder"
|
||||
>Force image ordering (prepend incremental numbers)
|
||||
</mat-checkbox
|
||||
>
|
||||
</mat-checkbox>
|
||||
|
||||
<mat-checkbox color="primary" formControlName="autoStart" name="autoStart"
|
||||
>Auto start downloads</mat-checkbox
|
||||
>
|
||||
|
||||
<mat-checkbox color="primary" formControlName="clearCompleted" name="clearCompleted"
|
||||
>Clear Finished
|
||||
</mat-checkbox>
|
||||
|
||||
<mat-slide-toggle color="primary" formControlName="vLogin" name="vLogin">
|
||||
ViperGirls Authentication
|
||||
</mat-slide-toggle>
|
||||
@@ -88,7 +101,11 @@
|
||||
(click)="onSubmit()"
|
||||
color="primary"
|
||||
type="submit"
|
||||
[disabled]="(generalSettingsForm.pristine && desktopSettingsForm.pristine) || generalSettingsForm.invalid || desktopSettingsForm.invalid"
|
||||
[disabled]="
|
||||
(generalSettingsForm.pristine && desktopSettingsForm.pristine) ||
|
||||
generalSettingsForm.invalid ||
|
||||
desktopSettingsForm.invalid
|
||||
"
|
||||
>
|
||||
Apply
|
||||
</button>
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { SettingsComponent } from './settings.component';
|
||||
|
||||
describe('SettingsComponent', () => {
|
||||
let component: SettingsComponent;
|
||||
let fixture: ComponentFixture<SettingsComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ SettingsComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(SettingsComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { AppService } from './../app.service';
|
||||
import { ClipboardService } from './../clipboard.service';
|
||||
import { Component, OnInit } from '@angular/core';
|
||||
import { Component, OnInit, ChangeDetectionStrategy } from '@angular/core';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { FormGroup, FormControl } from '@angular/forms';
|
||||
import { MatSnackBar } from '@angular/material';
|
||||
@@ -12,7 +12,8 @@ import { OpenDialogReturnValue } from 'electron';
|
||||
@Component({
|
||||
selector: 'app-settings',
|
||||
templateUrl: './settings.component.html',
|
||||
styleUrls: ['./settings.component.scss']
|
||||
styleUrls: ['./settings.component.scss'],
|
||||
changeDetection: ChangeDetectionStrategy.OnPush
|
||||
})
|
||||
export class SettingsComponent implements OnInit {
|
||||
constructor(
|
||||
@@ -29,10 +30,13 @@ export class SettingsComponent implements OnInit {
|
||||
maxThreads: new FormControl(''),
|
||||
autoStart: new FormControl(false),
|
||||
forceOrder: new FormControl(false),
|
||||
subLocation: new FormControl(false),
|
||||
threadSubLocation: new FormControl(false),
|
||||
clearCompleted: new FormControl(false),
|
||||
vLogin: new FormControl(false),
|
||||
vUsername: new FormControl(''),
|
||||
vPassword: new FormControl(''),
|
||||
vThanks: new FormControl(false)
|
||||
vThanks: new FormControl(false),
|
||||
});
|
||||
|
||||
desktopSettingsForm = new FormGroup({
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
<div class="status-bar">
|
||||
<span>{{ downloadSpeed!.speed + '/s' }}</span>
|
||||
<span>Downloading: {{ globalState!.running }}</span>
|
||||
<span>Queued: {{ globalState!.queued }}</span>
|
||||
<span>Remaining: {{ globalState!.remaining }}</span>
|
||||
<span>Error: {{ globalState!.error }}</span>
|
||||
<div class="status-bar" fxLayout="row">
|
||||
<span *ngIf="selected | async as length">Selected: {{length}}</span>
|
||||
<span fxFlex="grow"></span>
|
||||
<span *ngIf="downloadSpeed | async as speed">{{ speed!.speed + '/s' }}</span>
|
||||
<ng-container *ngIf="globalState | async as state">
|
||||
<span>Downloading: {{ state!.running }}</span>
|
||||
<span>Pending: {{ state!.queued }}</span>
|
||||
<span>Remaining: {{ state!.remaining }}</span>
|
||||
<span>Error: {{ state!.error }}</span>
|
||||
</ng-container>
|
||||
</div>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
width: 100%;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
text-align: end;
|
||||
white-space: nowrap;
|
||||
padding: 2px 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { StatusBarComponent } from './status-bar.component';
|
||||
|
||||
describe('StatusBarComponent', () => {
|
||||
let component: StatusBarComponent;
|
||||
let fixture: ComponentFixture<StatusBarComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [ StatusBarComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(StatusBarComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user