Compare commits

..
34 Commits
Author SHA1 Message Date
death-claw e8355b88b9 v3.2 2020-11-16 19:40:36 +01:00
death-claw 379660236d Commit missing files 2020-11-16 19:31:59 +01:00
death-claw c86ee91c27 v3.1.0 2020-11-16 16:44:26 +01:00
death-claw ff9d253242 UI revamp
Fix bugs
2020-11-16 16:42:30 +01:00
death-claw 53b45e00ba v3.0.11 2020-10-31 15:09:19 +01:00
death-claw 247b12a525 Add support for vipr.im 2020-10-31 15:06:26 +01:00
death-claw 7a8d332e6e v3.0.10 2020-10-04 16:31:57 +01:00
death-claw 749a6f18d2 Fix bug on settings update 2020-10-04 16:27:26 +01:00
death-claw 4107048f12 v3.0.9 2020-09-27 10:14:08 +01:00
death-claw f5d9ab85a2 Revert to appending post id to download folders
Fix a bug that lead to adding duplicate posts
Remove old unused settings
2020-09-27 10:12:29 +01:00
death-claw b82ee7b494 v3.0.8 2020-08-22 19:01:08 +01:00
death-claw 8a31dfc012 Bug fix which breaks concurrent downloads 2020-08-22 18:59:44 +01:00
death-claw 734c1b72c7 v3.0.7 2020-08-22 16:45:43 +01:00
death-claw 7abb7df1f6 Fix MacOs build 2020-08-22 16:44:24 +01:00
death-claw 0a7fa99a64 v3.0.6 2020-08-18 08:33:57 +01:00
death-claw 691026c8a9 Fix TurboImageHost 2020-08-18 08:32:33 +01:00
death-claw 3ef004b947 v3.0.5 2020-08-17 20:06:12 +01:00
death-claw 1b22a93e28 Update packages
Host class refactoring
2020-08-17 20:04:39 +01:00
death-claw 2639df0aaa Update packages
Host class refactoring
2020-08-17 20:04:23 +01:00
death-claw d5ae5d57d1 v3.0.4 2020-08-16 16:11:11 +01:00
death-claw 537d76d7be Improve rename UI 2020-08-16 16:06:17 +01:00
death-claw c1247faacc v3.0.3 2020-08-16 11:29:01 +01:00
death-claw efcc685f30 Remove notification feature 2020-08-16 11:26:57 +01:00
death-claw c825b22ce4 Fix some bugs with MacOS 2020-08-16 10:58:19 +01:00
death-claw 6ef5d5474f Fix some bugs with MacOS 2020-08-16 10:20:56 +01:00
death-claw bd25801c6c Fix base dir configuration for MacOS 2020-08-16 00:18:18 +01:00
death-claw 377b93c1ac Fix base dir configuration for AppImage 2020-08-16 00:02:18 +01:00
death-claw d498d20f6b Set default download path to user's home 2020-08-15 23:23:43 +01:00
death-claw d18544d414 Fix bin location for mac 2020-08-15 23:10:58 +01:00
death-claw 3c872b79bd Update app data configuration 2020-08-15 23:00:48 +01:00
death-claw 37900cac49 v3.0.2 2020-08-15 15:38:53 +01:00
death-claw be57c76d93 Fix bugs with download queue 2020-08-15 15:36:47 +01:00
death-claw 8b52ad7858 v3.0.1 2020-08-03 18:48:45 +01:00
death-claw a0976bde02 Fix bugs with download queue 2020-08-03 18:43:42 +01:00
184 changed files with 5606 additions and 5148 deletions
+56
View File
@@ -1,5 +1,61 @@
# Changelog
## [3.2] - 2020-11-16
### Changed
- Fix build issue
## [3.1.0] - 2020-11-16
### Changed
- UI revamp
- Fix bugs
## [3.0.11] - 2020-10-31
### Added
- Support for vipr.im
## [3.0.10] - 2020-10-04
### Changed
- Fix bug on settings update
## [3.0.9] - 2020-09-27
### Changed
- Revert to appending post id to download folders
- Fix a bug that lead to adding duplicate posts
- Remove old unused settings
## [3.0.8] - 2020-08-22
### Changed
- Bug fix which breaks concurrent downloads
## [3.0.7] - 2020-08-22
### Changed
- Fix MacOs build
## [3.0.6] - 2020-08-18
### Changed
- Fix TurboImageHost
## [3.0.5] - 2020-08-17
### Changed
- Update packages
- Host class refactoring
## [3.0.4] - 2020-08-16
### Changed
- Improve rename UI
## [3.0.3] - 2020-08-16
### Changed
- Fix bugs with electron app
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
+2 -2
View File
@@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.2.0</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
@@ -17,4 +17,4 @@
<module>vripper-ui</module>
<module>vripper-electron</module>
</modules>
</project>
</project>
+63 -50
View File
@@ -1,13 +1,16 @@
require('v8-compile-cache');
const { app, BrowserWindow } = require("electron");
const {app, BrowserWindow} = require("electron");
const path = require("path");
const url = require("url");
const getPort = require("get-port");
const { spawn } = require("child_process");
const { ipcMain } = require("electron");
const { dialog } = require("electron");
const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const appDir = process.env.APPDIR;
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
const appImagePath = process.env.APPIMAGE;
let win;
let vripperServer;
@@ -23,14 +26,14 @@ process.on("uncaughtException", err => {
process.exit(1);
});
function createWindow() {
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId("tn.mnlr.vripper");
}
let icon;
if(process.platform === "win32") {
if (process.platform === "win32") {
icon = __dirname + '/icon.ico';
} else {
icon = __dirname + '/icon.png';
@@ -66,6 +69,33 @@ function createWindow() {
});
}
shutdownServer = () => {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: {'content-type': 'application/json'},
}).then((response) => {
terminationInteval = setInterval(() => {
terminationAttemps++;
if (terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if (terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
}).catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
@@ -76,29 +106,30 @@ if (!gotTheLock) {
ipcMain.on("get-port", event => {
event.reply("port", port);
});
let javaBinPath;
if(appDir !== undefined) {
javaBinPath = path.join(appDir, "java-runtime/bin/java");
const appPath = path.join(app.getAppPath(), '../../');
let javaBinPath, jarPath, baseDir;
if (appImageDir !== undefined && process.platform === 'linux') {
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
baseDir = path.join(appImagePath, '..');
} else if (process.platform === 'darwin') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = app.getPath('appData');
} else if (process.platform === 'win32') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = appPath;
} else {
if(process.platform === 'darwin') {
javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
} else {
javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
}
}
let jarPath;
if(appDir !== undefined) {
jarPath = path.join(appDir, "bin/vripper-server.jar");
} else {
if(process.platform === 'darwin') {
jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
} else {
jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
}
console.error(`Unknown platform ${process.platform}`);
app.quit();
}
vripperServer = spawn(javaBinPath, [
"-Xms256m",
"-Dvripper.server.port=" + port,
"-Dbase.dir=" + baseDir,
"-jar",
jarPath
], {
@@ -122,31 +153,13 @@ if (!gotTheLock) {
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: { 'content-type': 'application/json' },
}).then((response) => {
terminationInteval= setInterval(() => {
terminationAttemps++;
if(terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if(terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
})
.catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
shutdownServer();
}
});
app.on("will-quit", () => {
if (process.platform === "darwin") {
shutdownServer();
}
});
+240 -255
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "3.0.0",
"version": "3.2.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@@ -64,9 +64,9 @@
"dev": true
},
"@types/fs-extra": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-B42Sxuaz09MhC3DDeW5kubRcQ5by4iuVQ0cRRWM2lggLzAa/KVom0Aft/208NgMvNQQZ86s5rVcqDdn/SH0/mg==",
"version": "9.0.2",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.2.tgz",
"integrity": "sha512-jp0RI6xfZpi5JL8v7WQwpBEQTq63RqW2kxwTZt+m27LcJqQdPVU1yGnT1ZI4EtCDynQQJtIGyQahkiCGCS7e+A==",
"dev": true,
"requires": {
"@types/node": "*"
@@ -78,9 +78,9 @@
"integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q=="
},
"@types/yargs": {
"version": "15.0.5",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.5.tgz",
"integrity": "sha512-Dk/IDOPtOgubt/IaevIUbTgV7doaKkoorvOyYM2CMwuDyP89bekI7H4xLIwunNYiK9jhCkmc6pUrJk3cj2AB9w==",
"version": "15.0.9",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.9.tgz",
"integrity": "sha512-HmU8SeIRhZCWcnRskCs36Q1Q00KBV6Cqh/ora8WN1+22dY07AZdn6Gel8QZ3t26XYPImtcL8WV/eqjhVmMEw4g==",
"dev": true,
"requires": {
"@types/yargs-parser": "*"
@@ -93,9 +93,9 @@
"dev": true
},
"ajv": {
"version": "6.12.3",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz",
"integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==",
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
"requires": {
"fast-deep-equal": "^3.1.1",
@@ -105,9 +105,9 @@
}
},
"ajv-keywords": {
"version": "3.5.1",
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.1.tgz",
"integrity": "sha512-KWcq3xN8fDjSB+IMoh2VaXVhRI0BBGxoYp3rx7Pkb6z0cFjYR9Q9l4yZqqals0/zsioCmocC5H6UvsGD4MoIBA==",
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
"integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
"dev": true
},
"ansi-align": {
@@ -148,29 +148,29 @@
}
},
"app-builder-bin": {
"version": "3.5.9",
"resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-3.5.9.tgz",
"integrity": "sha512-NSjtqZ3x2kYiDp3Qezsgukx/AUzKPr3Xgf9by4cYt05ILWGAptepeeu0Uv+7MO+41o6ujhLixTou8979JGg2Kg==",
"version": "3.5.10",
"resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-3.5.10.tgz",
"integrity": "sha512-Jd+GW68lR0NeetgZDo47PdWBEPdnD+p0jEa7XaxjRC8u6Oo/wgJsfKUkORRgr2NpkD19IFKN50P6JYy04XHFLQ==",
"dev": true
},
"app-builder-lib": {
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-22.7.0.tgz",
"integrity": "sha512-blRKwV8h0ztualXS50ciCTo39tbuDGNS+ldcy8+KLvKXuT6OpYnSJ7M6MSfPT+xWatshMHJV1rJx3Tl+k/Sn/g==",
"version": "22.9.1",
"resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-22.9.1.tgz",
"integrity": "sha512-KfXim/fiNwFW2SKffsjEMdAU7RbbEXn62x5YyXle1b4j9X/wEHW9iwox8De6y0hJdR+/kCC/49lI+VgNwLhV7A==",
"dev": true,
"requires": {
"7zip-bin": "~5.0.3",
"@develar/schema-utils": "~2.6.5",
"async-exit-hook": "^2.0.1",
"bluebird-lst": "^1.0.9",
"builder-util": "22.7.0",
"builder-util-runtime": "8.7.1",
"builder-util": "22.9.1",
"builder-util-runtime": "8.7.2",
"chromium-pickle-js": "^0.2.0",
"debug": "^4.2.0",
"ejs": "^3.1.3",
"electron-publish": "22.7.0",
"fs-extra": "^9.0.0",
"hosted-git-info": "^3.0.4",
"debug": "^4.3.0",
"ejs": "^3.1.5",
"electron-publish": "22.9.1",
"fs-extra": "^9.0.1",
"hosted-git-info": "^3.0.5",
"is-ci": "^2.0.0",
"isbinaryfile": "^4.0.6",
"js-yaml": "^3.14.0",
@@ -184,9 +184,9 @@
},
"dependencies": {
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.0.tgz",
"integrity": "sha512-jjO6JD2rKfiZQnBoRzhRTbXjHLGLfH+UtGkWLc/UXAh/rzZMyjbgn0NcfFpqT8nd1kTtFnDiJcrIFkq4UKeJVg==",
"dev": true,
"requires": {
"ms": "2.1.2"
@@ -205,13 +205,21 @@
}
},
"jsonfile": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
"integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
"dev": true,
"requires": {
"graceful-fs": "^4.1.6",
"universalify": "^1.0.0"
"universalify": "^2.0.0"
},
"dependencies": {
"universalify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
"dev": true
}
}
},
"universalify": {
@@ -358,20 +366,20 @@
"dev": true
},
"builder-util": {
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.7.0.tgz",
"integrity": "sha512-UV3MKL0mwjMq2y9JlBf28Cegpj0CrIXcjGkO0TXn+QZ6Yy9rY6lHOuUvpQ19ct2Qh1o+QSwH3Q1nKUf5viJBBg==",
"version": "22.9.1",
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.9.1.tgz",
"integrity": "sha512-5hN/XOaYu4ZQUS6F+5CXE6jTo+NAnVqAxDuKGSaHWb9bejfv/rluChTLoY3/nJh7RFjkoyVjvFJv7zQDB1QmHw==",
"dev": true,
"requires": {
"7zip-bin": "~5.0.3",
"@types/debug": "^4.1.5",
"@types/fs-extra": "^9.0.1",
"app-builder-bin": "3.5.9",
"app-builder-bin": "3.5.10",
"bluebird-lst": "^1.0.9",
"builder-util-runtime": "8.7.1",
"chalk": "^4.0.0",
"debug": "^4.2.0",
"fs-extra": "^9.0.0",
"builder-util-runtime": "8.7.2",
"chalk": "^4.1.0",
"debug": "^4.3.0",
"fs-extra": "^9.0.1",
"is-ci": "^2.0.0",
"js-yaml": "^3.14.0",
"source-map-support": "^0.5.19",
@@ -380,9 +388,9 @@
},
"dependencies": {
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.0.tgz",
"integrity": "sha512-jjO6JD2rKfiZQnBoRzhRTbXjHLGLfH+UtGkWLc/UXAh/rzZMyjbgn0NcfFpqT8nd1kTtFnDiJcrIFkq4UKeJVg==",
"dev": true,
"requires": {
"ms": "2.1.2"
@@ -401,13 +409,21 @@
}
},
"jsonfile": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
"integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
"dev": true,
"requires": {
"graceful-fs": "^4.1.6",
"universalify": "^1.0.0"
"universalify": "^2.0.0"
},
"dependencies": {
"universalify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
"dev": true
}
}
},
"universalify": {
@@ -419,24 +435,13 @@
}
},
"builder-util-runtime": {
"version": "8.7.1",
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-8.7.1.tgz",
"integrity": "sha512-uEBH1nAnTvzjcsrh2XI3qOzJ39h0+9kuIuwj+kCc3a07TZNGShfJcai8fFzL3mNgGjEFxoq+XMssR11r+FOFSg==",
"version": "8.7.2",
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-8.7.2.tgz",
"integrity": "sha512-xBqv+8bg6cfnzAQK1k3OGpfaHg+QkPgIgpEkXNhouZ0WiUkyZCftuRc2LYzQrLucFywpa14Xbc6+hTbpq83yRA==",
"dev": true,
"requires": {
"debug": "^4.2.0",
"debug": "^4.1.1",
"sax": "^1.2.4"
},
"dependencies": {
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"dev": true,
"requires": {
"ms": "2.1.2"
}
}
}
},
"cacheable-request": {
@@ -455,9 +460,9 @@
},
"dependencies": {
"get-stream": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
"integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
"dev": true,
"requires": {
"pump": "^3.0.0"
@@ -513,9 +518,9 @@
"dev": true
},
"cli-boxes": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.0.tgz",
"integrity": "sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==",
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz",
"integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==",
"dev": true
},
"cli-truncate": {
@@ -528,14 +533,14 @@
}
},
"cliui": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz",
"integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==",
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.3.tgz",
"integrity": "sha512-Gj3QHTkVMPKqwP3f7B4KPkBZRMR9r4rfi5bXFpg1a+Svvj8l7q5CnkBkVQzfxT5DFSsGk2+PascOgL0JYkL2kw==",
"dev": true,
"requires": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^6.2.0"
"wrap-ansi": "^7.0.0"
},
"dependencies": {
"ansi-regex": {
@@ -660,20 +665,14 @@
"integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
},
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"dev": true,
"requires": {
"ms": "^2.1.1"
"ms": "2.1.2"
}
},
"decamelize": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
"integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
"dev": true
},
"decompress-response": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
@@ -713,15 +712,15 @@
"optional": true
},
"dmg-builder": {
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-22.7.0.tgz",
"integrity": "sha512-5Ea2YEz6zSNbyGzZD+O9/MzmaXb6oa15cSKWo4JQ1xP4rorOpte7IOj2jcwYjtc+Los2gu1lvT314OC1OZIWgg==",
"version": "22.9.1",
"resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-22.9.1.tgz",
"integrity": "sha512-jc+DAirqmQrNT6KbDHdfEp8D1kD0DBTnsLhwUR3MX+hMBun5bT134LQzpdK0GKvd22GqF8L1Cz/NOgaVjscAXQ==",
"dev": true,
"requires": {
"app-builder-lib": "22.7.0",
"builder-util": "22.7.0",
"fs-extra": "^9.0.0",
"iconv-lite": "^0.5.1",
"app-builder-lib": "22.9.1",
"builder-util": "22.9.1",
"fs-extra": "^9.0.1",
"iconv-lite": "^0.6.2",
"js-yaml": "^3.14.0",
"sanitize-filename": "^1.6.3"
},
@@ -739,13 +738,21 @@
}
},
"jsonfile": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
"integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
"dev": true,
"requires": {
"graceful-fs": "^4.1.6",
"universalify": "^1.0.0"
"universalify": "^2.0.0"
},
"dependencies": {
"universalify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
"dev": true
}
}
},
"universalify": {
@@ -788,9 +795,9 @@
}
},
"dot-prop": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
"integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
"integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==",
"dev": true,
"requires": {
"is-obj": "^2.0.0"
@@ -815,18 +822,18 @@
"dev": true
},
"ejs": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.3.tgz",
"integrity": "sha512-wmtrUGyfSC23GC/B1SMv2ogAUgbQEtDmTIhfqielrG5ExIM9TP4UoYdi90jLF1aTcsWCJNEO0UrgKzP0y3nTSg==",
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.5.tgz",
"integrity": "sha512-dldq3ZfFtgVTJMLjOe+/3sROTzALlL9E34V4/sDtUd/KlBSS0s6U1/+WPE1B4sj9CXHJpL1M6rhNJnc9Wbal9w==",
"dev": true,
"requires": {
"jake": "^10.6.1"
}
},
"electron": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/electron/-/electron-9.1.0.tgz",
"integrity": "sha512-VRAF8KX1m0py9I9sf0kw1kWfeC87mlscfFcbcRdLBsNJ44/GrJhi3+E8rKbpHUeZNQxsPaVA5Zu5Lxb6dV/scQ==",
"version": "10.1.5",
"resolved": "https://registry.npmjs.org/electron/-/electron-10.1.5.tgz",
"integrity": "sha512-fys/KnEfJq05TtMij+lFvLuKkuVH030CHYx03iZrW5DNNLwjE6cW3pysJ420lB0FRSfPjTHBMu2eVCf5TG71zQ==",
"dev": true,
"requires": {
"@electron/get": "^1.0.1",
@@ -835,33 +842,33 @@
},
"dependencies": {
"@types/node": {
"version": "12.12.50",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.50.tgz",
"integrity": "sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w==",
"version": "12.19.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.19.3.tgz",
"integrity": "sha512-8Jduo8wvvwDzEVJCOvS/G6sgilOLvvhn1eMmK3TW8/T217O7u1jdrK6ImKLv80tVryaPSVeKu6sjDEiFjd4/eg==",
"dev": true
}
}
},
"electron-builder": {
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-22.7.0.tgz",
"integrity": "sha512-t6E3oMutpST64YWbZCg7HodEwJOsnjUF1vnDIHm2MW6CFZPX8tlCK6efqaV66LU0E0Nkp/JH6TE5bCqQ1+VdPQ==",
"version": "22.9.1",
"resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-22.9.1.tgz",
"integrity": "sha512-GXPt8l5Mxwm1QKYopUM6/Tdh9W3695G6Ax+IFyj5pQ51G4SD5L1uq4/RkPSsOgs3rP7jNSV6g6OfDzdtVufPdA==",
"dev": true,
"requires": {
"@types/yargs": "^15.0.5",
"app-builder-lib": "22.7.0",
"app-builder-lib": "22.9.1",
"bluebird-lst": "^1.0.9",
"builder-util": "22.7.0",
"builder-util-runtime": "8.7.1",
"chalk": "^4.0.0",
"dmg-builder": "22.7.0",
"fs-extra": "^9.0.0",
"builder-util": "22.9.1",
"builder-util-runtime": "8.7.2",
"chalk": "^4.1.0",
"dmg-builder": "22.9.1",
"fs-extra": "^9.0.1",
"is-ci": "^2.0.0",
"lazy-val": "^1.0.4",
"read-config-file": "6.0.0",
"sanitize-filename": "^1.6.3",
"update-notifier": "^4.1.0",
"yargs": "^15.3.1"
"update-notifier": "^4.1.1",
"yargs": "^16.0.3"
},
"dependencies": {
"fs-extra": {
@@ -877,13 +884,21 @@
}
},
"jsonfile": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
"integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
"dev": true,
"requires": {
"graceful-fs": "^4.1.6",
"universalify": "^1.0.0"
"universalify": "^2.0.0"
},
"dependencies": {
"universalify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
"dev": true
}
}
},
"universalify": {
@@ -920,19 +935,19 @@
"integrity": "sha512-R1oD5gMBPS7PVU8gJwH6CtT0e6VSoD0+SzSnYpNm+dBkcijgA+K7VAMHDfnRq/lkKPZArpzplTW6jfiMYosdzw=="
},
"electron-publish": {
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-22.7.0.tgz",
"integrity": "sha512-hmU69xlb6vvAV3QfpHYDlkdZMFdBAgDbptoxbLFrnTq5bOkcL8AaDbvxeoZ4+lvqgs29NwqGpkHo2oN+p/hCfg==",
"version": "22.9.1",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-22.9.1.tgz",
"integrity": "sha512-ducLjRJLEeU87FaTCWaUyDjCoLXHkawkltP2zqS/n2PyGke54ZIql0tBuUheht4EpR8AhFbVJ11spSn1gy8r6w==",
"dev": true,
"requires": {
"@types/fs-extra": "^9.0.1",
"bluebird-lst": "^1.0.9",
"builder-util": "22.7.0",
"builder-util-runtime": "8.7.1",
"chalk": "^4.0.0",
"fs-extra": "^9.0.0",
"builder-util": "22.9.1",
"builder-util-runtime": "8.7.2",
"chalk": "^4.1.0",
"fs-extra": "^9.0.1",
"lazy-val": "^1.0.4",
"mime": "^2.4.5"
"mime": "^2.4.6"
},
"dependencies": {
"fs-extra": {
@@ -948,13 +963,21 @@
}
},
"jsonfile": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
"integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
"dev": true,
"requires": {
"graceful-fs": "^4.1.6",
"universalify": "^1.0.0"
"universalify": "^2.0.0"
},
"dependencies": {
"universalify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
"dev": true
}
}
},
"universalify": {
@@ -1005,6 +1028,12 @@
"dev": true,
"optional": true
},
"escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
"dev": true
},
"escape-goat": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz",
@@ -1099,16 +1128,6 @@
"minimatch": "^3.0.4"
}
},
"find-up": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
"integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
"dev": true,
"requires": {
"locate-path": "^5.0.0",
"path-exists": "^4.0.0"
}
},
"follow-redirects": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
@@ -1148,6 +1167,12 @@
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
"dev": true
},
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -1254,6 +1279,15 @@
"integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==",
"dev": true
},
"has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
"integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
"dev": true,
"requires": {
"function-bind": "^1.1.1"
}
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -1267,9 +1301,9 @@
"dev": true
},
"hosted-git-info": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.5.tgz",
"integrity": "sha512-i4dpK6xj9BIpVOTboXIlKG9+8HMKggcrMX7WA24xZtKwX0TPelq/rbaS5rCKeNX8sJXZJGdSxpnEGtta+wismQ==",
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.7.tgz",
"integrity": "sha512-fWqc0IcuXs+BmE9orLDyVykAG9GJtGLGuZAAqgcckPgv5xad4AcXGIv8galtQvlwutxSlaMcdw7BUtq2EIvqCQ==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
@@ -1315,12 +1349,12 @@
"dev": true
},
"iconv-lite": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz",
"integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==",
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
"integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
"dev": true,
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
"safer-buffer": ">= 2.1.2 < 3.0.0"
}
},
"import-lazy": {
@@ -1364,6 +1398,15 @@
"ci-info": "^2.0.0"
}
},
"is-core-module": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.0.0.tgz",
"integrity": "sha512-jq1AH6C8MuteOoBPwkxHafmByhL9j5q4OaPGdbuD+ZtQJVzH+i6E3BJDQcBA09k57i2Hh2yQbEG8yObZ0jdlWw==",
"dev": true,
"requires": {
"has": "^1.0.3"
}
},
"is-fullwidth-code-point": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
@@ -1568,19 +1611,10 @@
"integrity": "sha512-u93kb2fPbIrfzBuLjZE+w+fJbUUMhNDXxNmMfaqNgpfQf1CO5ZSe2LfsnBqVAk7i/2NF48OSoRj+Xe2VT+lE8Q==",
"dev": true
},
"locate-path": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
"integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
"dev": true,
"requires": {
"p-locate": "^4.1.0"
}
},
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
"version": "4.17.20",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"lowercase-keys": {
"version": "1.0.1",
@@ -1747,30 +1781,6 @@
"integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==",
"dev": true
},
"p-limit": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
"dev": true,
"requires": {
"p-try": "^2.0.0"
}
},
"p-locate": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
"dev": true,
"requires": {
"p-limit": "^2.2.0"
}
},
"p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
"dev": true
},
"package-json": {
"version": "6.5.0",
"resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz",
@@ -1918,9 +1928,9 @@
}
},
"registry-auth-token": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.1.1.tgz",
"integrity": "sha512-9bKS7nTl9+/A1s7tnPeGrUpRcVY+LUh7bfFgzpndALdPfXQBfQV77rQVtqgUV3ti4vc/Ik81Ex8UJDWDQ12zQA==",
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.0.tgz",
"integrity": "sha512-P+lWzPrsgfN+UEpDS3U8AQKg/UjZX6mQSJueZj3EK+vNESoqBSpBUD3gmu4sF9lOsjXWjF11dQKUqemf3veq1w==",
"dev": true,
"requires": {
"rc": "^1.2.8"
@@ -1941,18 +1951,13 @@
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"dev": true
},
"require-main-filename": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
"integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
"dev": true
},
"resolve": {
"version": "1.17.0",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz",
"integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==",
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.18.1.tgz",
"integrity": "sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==",
"dev": true,
"requires": {
"is-core-module": "^2.0.0",
"path-parse": "^1.0.6"
}
},
@@ -1974,13 +1979,13 @@
}
},
"roarr": {
"version": "2.15.3",
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.3.tgz",
"integrity": "sha512-AEjYvmAhlyxOeB9OqPUzQCo3kuAkNfuDk/HqWbZdFsqDFpapkTjiw+p4svNEoRLvuqNTxqfL+s+gtD4eDgZ+CA==",
"version": "2.15.4",
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
"integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
"dev": true,
"optional": true,
"requires": {
"boolean": "^3.0.0",
"boolean": "^3.0.1",
"detect-node": "^2.0.4",
"globalthis": "^1.0.1",
"json-stringify-safe": "^5.0.1",
@@ -2054,12 +2059,6 @@
"type-fest": "^0.13.1"
}
},
"set-blocking": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
"dev": true
},
"signal-exit": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
@@ -2142,9 +2141,9 @@
}
},
"spdx-license-ids": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz",
"integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==",
"version": "3.0.6",
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.6.tgz",
"integrity": "sha512-+orQK83kyMva3WyPf59k1+Y525csj5JejicWut55zeTWANuN17qSiSLUXWtzHeNWORSvT7GLDJ/E/XiIWoXBTw==",
"dev": true
},
"sprintf-js": {
@@ -2229,9 +2228,9 @@
}
},
"supports-color": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz",
"integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==",
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"requires": {
"has-flag": "^4.0.0"
@@ -2248,9 +2247,9 @@
}
},
"term-size": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.0.tgz",
"integrity": "sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==",
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz",
"integrity": "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==",
"dev": true
},
"to-readable-stream": {
@@ -2322,9 +2321,9 @@
}
},
"update-notifier": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.0.tgz",
"integrity": "sha512-w3doE1qtI0/ZmgeoDoARmI5fjDoT93IfKgEGqm26dGUOh8oNpaSTsGNdYRN/SjOuo10jcJGwkEL3mroKzktkew==",
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz",
"integrity": "sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A==",
"dev": true,
"requires": {
"boxen": "^4.2.0",
@@ -2355,9 +2354,9 @@
}
},
"uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz",
"integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==",
"dev": true,
"requires": {
"punycode": "^2.1.0"
@@ -2398,12 +2397,6 @@
"spdx-expression-parse": "^3.0.0"
}
},
"which-module": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
"integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
"dev": true
},
"widest-line": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz",
@@ -2414,9 +2407,9 @@
}
},
"wrap-ansi": {
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
"integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dev": true,
"requires": {
"ansi-styles": "^4.0.0",
@@ -2465,9 +2458,9 @@
"dev": true
},
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
"integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==",
"dev": true
},
"yallist": {
@@ -2477,33 +2470,25 @@
"dev": true
},
"yargs": {
"version": "15.4.1",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
"integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
"version": "16.1.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.1.0.tgz",
"integrity": "sha512-upWFJOmDdHN0syLuESuvXDmrRcWd1QafJolHskzaw79uZa7/x53gxQKiR07W59GWY1tFhhU/Th9DrtSfpS782g==",
"dev": true,
"requires": {
"cliui": "^6.0.0",
"decamelize": "^1.2.0",
"find-up": "^4.1.0",
"get-caller-file": "^2.0.1",
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"require-main-filename": "^2.0.0",
"set-blocking": "^2.0.0",
"string-width": "^4.2.0",
"which-module": "^2.0.0",
"y18n": "^4.0.0",
"yargs-parser": "^18.1.2"
"y18n": "^5.0.2",
"yargs-parser": "^20.2.2"
}
},
"yargs-parser": {
"version": "18.1.3",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
"integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
"dev": true,
"requires": {
"camelcase": "^5.0.0",
"decamelize": "^1.2.0"
}
"version": "20.2.3",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.3.tgz",
"integrity": "sha512-emOFRT9WVHw03QSvN5qor9QQT9+sw5vwxfYweivSMHTcAXPefwVae2FjO7JJjj8hCE4CzPOPeFM83VwT29HCww==",
"dev": true
},
"yauzl": {
"version": "2.10.0",
+4 -9
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "3.0.0",
"version": "3.2.0",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -40,14 +40,9 @@
"win": {
"icon": "icon.ico",
"target": [
"nsis"
"dir"
]
},
"nsis": {
"createDesktopShortcut": "always",
"oneClick": false,
"perMachine": false
},
"linux": {
"synopsis": "vipergirls.to ripper",
"category": "Utility",
@@ -70,8 +65,8 @@
"dist": "node pre-build.js && electron-builder"
},
"devDependencies": {
"electron": "9.1.0",
"electron-builder": "22.7.0"
"electron": "10.1.5",
"electron-builder": "22.9.1"
},
"dependencies": {
"axios": "0.19.2",
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.2.0</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.2.0</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
@@ -6,17 +6,14 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
@SpringBootApplication
@Slf4j
public class VripperApplication {
public static final RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
.handleIf(e -> !(e instanceof InterruptedException))
.withDelay(1, 3, ChronoUnit.SECONDS)
.withMaxAttempts(5)
.abortOn(Collections.singletonList(InterruptedException.class))
.onFailedAttempt(e -> log.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
public static void main(String[] args) {
@@ -27,6 +24,5 @@ public class VripperApplication {
log.error("Failed to run the application", e);
}
}
}
@@ -0,0 +1,288 @@
package tn.mnlr.vripper.download;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.function.CheckedRunnable;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.*;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public class DownloadJob implements CheckedRunnable {
public enum ContextAttributes {
OPEN_CONNECTION("OPEN_CONNECTION");
private final String value;
ContextAttributes(String value) {
this.value = value;
}
@Override
public String toString() {
return value;
}
}
private static final ReentrantLock LOCK = new ReentrantLock();
private static final int READ_BUFFER_SIZE = 8192;
private final DataService dataService;
private final PathService pathService;
private final ConnectionService cm;
private final VGAuthService authService;
private final DownloadSpeedService downloadSpeedService;
private final SettingsService settingsService;
private final HttpClientContext context;
@Getter
private final Image image;
@Getter
private final Post post;
private boolean stopped = false;
@Getter
private boolean finished = false;
DownloadJob(Post post, Image image) {
this.image = image;
this.post = post;
dataService = SpringContext.getBean(DataService.class);
pathService = SpringContext.getBean(PathService.class);
cm = SpringContext.getBean(ConnectionService.class);
authService = SpringContext.getBean(VGAuthService.class);
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
settingsService = SpringContext.getBean(SettingsService.class);
context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
context.setAttribute(ContextAttributes.OPEN_CONNECTION.toString(), Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
}
public void download(final Post post, final Image image) throws DownloadException {
try {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
try {
LOCK.lock();
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
} finally {
LOCK.unlock();
}
if (stopped) {
return;
}
/*
* HOST SPECIFIC
*/
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
/*
* END HOST SPECIFIC
*/
if (stopped) {
return;
}
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
log.debug(String.format("Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
HttpClient client = cm.getClient().build();
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
httpGet.addHeader("Referer", image.getUrl());
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (stopped) {
return;
}
File destinationFolder, outputFile;
try {
LOCK.lock();
Post updatedPost = dataService.findPostById(post.getId()).orElseThrow();
if (updatedPost.getPostFolderName() == null) {
pathService.createDefaultPostFolder(updatedPost);
}
destinationFolder = pathService.getDownloadDestinationFolder(updatedPost);
if (settingsService.getSettings().getLeaveThanksOnStart()) {
authService.leaveThanks(updatedPost);
}
outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + nameAndUrl.getName() + ".tmp");
outputFile.getParentFile().mkdirs();
} finally {
LOCK.unlock();
}
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
if (stopped) {
return;
}
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
if (stopped) {
return;
}
}
checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
}
} catch (Exception e) {
if (stopped) {
return;
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
image.setStatus(Status.COMPLETE);
} else if (stopped) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
finished = true;
}
}
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
if (reader.getFormatName().toUpperCase().equals("JPEG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
imageName += ".jpg";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_jpg")) {
toReplace = "_jpg";
} else if (imageNameLC.endsWith("_jpeg")) {
toReplace = "_jpeg";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
}
}
} else if (reader.getFormatName().toUpperCase().equals("PNG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_png")) {
imageName += ".png";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_png")) {
toReplace = "_png";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
}
}
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
File downloadDestinationFolder = pathService.getDownloadDestinationFolder(post);
File outImage = new File(downloadDestinationFolder, (settingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
if (outImage.exists() && outImage.delete()) {
log.debug(String.format("%s is deleted", outImage.toString()));
}
Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
}
}
@Override
public void run() throws Exception {
if (stopped) {
finished = true;
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
download(post, image);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return Objects.equals(image, that.image) &&
Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image, post);
}
public void stop() {
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
this.stopped = true;
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.q;
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
@@ -6,48 +6,32 @@ import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.MutexService;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public class ExecuteRunnable implements Runnable {
public class DownloadRunnable implements Runnable {
private final ExecutionService executionService;
private final DownloadService downloadService;
private final DataService dataService;
private final MutexService mutexService;
private final DownloadJob downloadJob;
public ExecuteRunnable(final DownloadJob downloadJob) {
executionService = SpringContext.getBean(ExecutionService.class);
public DownloadRunnable(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
mutexService = SpringContext.getBean(MutexService.class);
this.downloadJob = downloadJob;
}
@Override
public void run() {
mutexService.createPostLock(downloadJob.getPost().getPostId());
ReentrantLock mutex = mutexService.getPostLock(downloadJob.getPost().getPostId());
mutex.lock();
downloadJob.refresh();
executionService.beforeJobStart(downloadJob.getPost().getPostId());
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> {
if (e.getFailure() instanceof InterruptedException || e.getFailure().getCause() instanceof InterruptedException) {
log.debug("Job successfully interrupted");
return;
}
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
})
.onComplete(e -> {
mutex.unlock();
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
executionService.afterJobFinish(downloadJob);
downloadService.afterJobFinish(downloadJob);
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
}).run(downloadJob);
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.q;
package tn.mnlr.vripper.download;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@@ -8,22 +8,19 @@ import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.MutexService;
import tn.mnlr.vripper.services.post.PostService;
import tn.mnlr.vripper.services.PostService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@Slf4j
public class ExecutionService {
public class DownloadService {
private static final List<Status> FINISHED = Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
private final int MAX_POOL_SIZE = 12;
@@ -31,26 +28,23 @@ public class ExecutionService {
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
private final Map<DownloadJob, Future<?>> futures = new ConcurrentHashMap<>();
private final Map<String, AtomicInteger> downloadCount = new ConcurrentHashMap<>();
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
private final PendingQ pendingQ;
private final AppSettingsService settings;
private final PendingQueue pendingQueue;
private final SettingsService settings;
private final DataService dataService;
private final PostService postService;
private final MutexService mutexService;
private boolean pauseQ = false;
private Thread executionThread;
private Thread pollThread;
@Autowired
public ExecutionService(PendingQ pendingQ, AppSettingsService settings, DataService dataService, PostService postService, MutexService mutexService) {
this.pendingQ = pendingQ;
public DownloadService(PendingQueue pendingQueue, SettingsService settings, DataService dataService, PostService postService) {
this.pendingQueue = pendingQueue;
this.settings = settings;
this.dataService = dataService;
this.postService = postService;
this.mutexService = mutexService;
}
@PostConstruct
@@ -61,8 +55,7 @@ public class ExecutionService {
executionThread.start();
}
@PreDestroy
private void destroy() throws InterruptedException {
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
executionThread.interrupt();
pollThread.interrupt();
@@ -71,22 +64,29 @@ public class ExecutionService {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
executor.awaitTermination(10, TimeUnit.SECONDS);
executor.awaitTermination(5, TimeUnit.SECONDS);
}
private void stopRunning(@NonNull String postId) {
futures
.entrySet()
.stream()
.filter(e -> e.getKey().getImage().getPostId().equals(postId))
.forEach(e -> {
e.getValue().cancel(true);
if (e.getKey().getImageFileData().getImageRequest() != null) {
e.getKey().getImageFileData().getImageRequest().abort();
}
e.getKey().getImage().setStatus(Status.STOPPED);
dataService.updateImageStatus(e.getKey().getImage().getStatus(), e.getKey().getImage().getId());
});
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = executing.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> posIds) {
@@ -106,7 +106,7 @@ public class ExecutionService {
}
private void restart(@NonNull String postId) {
if (isRunning(postId) || isPending(postId)) {
if (isPending(postId)) {
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
@@ -120,7 +120,7 @@ public class ExecutionService {
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
try {
pendingQ.put(post, image);
pendingQueue.put(post, image);
} catch (InterruptedException e) {
log.warn("Thread was interrupted", e);
Thread.currentThread().interrupt();
@@ -129,12 +129,7 @@ public class ExecutionService {
}
private boolean isPending(String postId) {
return pendingQ.isPending(postId);
}
public boolean isRunning(@NonNull final String postId) {
AtomicInteger runningCount = downloadCount.get(postId);
return runningCount != null && runningCount.get() > 0;
return pendingQueue.isPending(postId);
}
private void stop(String postId) {
@@ -147,9 +142,10 @@ public class ExecutionService {
if (FINISHED.contains(post.getStatus())) {
return;
}
pendingQ.remove(post);
pendingQueue.stop(post);
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
postService.stopFetchingMetadata(post);
} finally {
pauseQ = false;
@@ -173,11 +169,11 @@ public class ExecutionService {
private void poll() {
while (!Thread.interrupted()) {
try {
List<DownloadJob> peek = pendingQ.peek();
List<DownloadJob> peek = pendingQueue.peek();
for (DownloadJob downloadJob : peek) {
if (canRun(downloadJob.getImage().getHost())) {
executionQueue.offer(downloadJob);
pendingQ.remove(downloadJob);
pendingQueue.remove(downloadJob);
}
}
synchronized (threadCount) {
@@ -205,37 +201,24 @@ public class ExecutionService {
}
private void push(DownloadJob downloadJob) {
ExecuteRunnable runnable = new ExecuteRunnable(downloadJob);
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
futures.put(downloadJob, executor.submit(runnable));
}
public void beforeJobStart(String postId) {
checkKeyRunningPosts(postId);
downloadCount.get(postId).incrementAndGet();
}
private synchronized void checkKeyRunningPosts(@NonNull final String postId) {
if (!downloadCount.containsKey(postId)) {
downloadCount.put(postId, new AtomicInteger(0));
}
executor.execute(new DownloadRunnable(downloadJob));
executing.add(downloadJob);
}
public synchronized void afterJobFinish(DownloadJob downloadJob) {
int count = downloadCount.get(downloadJob.getPost().getPostId()).decrementAndGet();
if (count == 0 && !pendingQ.isPending(downloadJob.getPost().getPostId())) {
downloadCount.remove(downloadJob.getPost().getPostId());
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
if (count == 0) {
dataService.finishPost(downloadJob.getPost());
mutexService.removePostLock(downloadJob.getPost().getPostId());
}
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
futures.remove(downloadJob);
executing.remove(downloadJob);
synchronized (threadCount) {
threadCount.notify();
}
}
public int runningCount() {
return futures.size();
return executing.size();
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.q;
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -6,7 +6,7 @@ import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.DataService;
import javax.annotation.PostConstruct;
@@ -14,21 +14,24 @@ import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
@Service
@Slf4j
public class PendingQ {
public class PendingQueue {
private final DataService dataService;
private final AppSettingsService appSettingsService;
private final SettingsService settingsService;
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
@Autowired
public PendingQ(DataService dataService, AppSettingsService appSettingsService, List<Host> hosts) {
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
this.dataService = dataService;
this.appSettingsService = appSettingsService;
this.settingsService = settingsService;
this.hosts = hosts;
}
@@ -44,6 +47,14 @@ public class PendingQ {
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob = new DownloadJob(post, image);
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
checkKey(post.getPostId());
toBeExecuted.get(post.getPostId()).incrementAndGet();
}
private synchronized void checkKey(String postId) {
if (!toBeExecuted.containsKey(postId)) {
toBeExecuted.put(postId, new AtomicInteger(0));
}
}
public void remove(final DownloadJob downloadJob) {
@@ -57,7 +68,7 @@ public class PendingQ {
}
for (Host host : hosts) {
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
for (int i = 0; i < appSettingsService.getSettings().getMaxThreads(); i++) {
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
if (downloadJob != null) {
downloadJobs.add(downloadJob);
@@ -74,17 +85,31 @@ public class PendingQ {
}
public int size() {
return pendingQ.values().stream().mapToInt(BlockingDeque::size).sum();
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
}
public void remove(Post post) {
public void stop(Post post) {
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
entry.getValue().removeIf(next -> next.getImage().getPostId().equals(post.getPostId()));
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
entry.getValue().removeIf(predicate);
decrement(post.getPostId());
}
dataService.finishPost(post);
}
public boolean isPending(String postId) {
return pendingQ.values().stream().flatMap(Collection::stream).anyMatch(e -> e.getPost().getPostId().equals(postId));
return toBeExecuted.containsKey(postId);
}
public synchronized int decrement(String postId) {
AtomicInteger counter = toBeExecuted.get(postId);
if (counter == null) {
return 0;
}
int count = counter.decrementAndGet();
if (count == 0) {
toBeExecuted.remove(postId);
}
return count;
}
}
@@ -15,8 +15,10 @@ import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.util.ArrayList;
import java.util.List;
@@ -29,11 +31,18 @@ public class AcidimgHost extends Host {
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
private static final String IMG_XPATH = "//img[@class='centred']";
@Autowired
private ConnectionManager cm;
private final ConnectionService cm;
private final HostService hostService;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
public AcidimgHost() {
super();
@Autowired
public AcidimgHost(ConnectionService cm, HostService hostService, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.hostService = hostService;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
@@ -47,9 +56,9 @@ public class AcidimgHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
Node contDiv;
try {
@@ -62,7 +71,7 @@ public class AcidimgHost extends Host {
if (contDiv != null) {
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
HttpPost httpPost = cm.buildHttpPost(url, context);
httpPost.addHeader("Referer", url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
@@ -99,8 +108,7 @@ public class AcidimgHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? getDefaultImageName(imgUrl) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -1,76 +1,17 @@
package tn.mnlr.vripper.host;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.HostService;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Iterator;
import java.util.Objects;
@Service
@Slf4j
abstract public class Host {
private static final int READ_BUFFER_SIZE = 8192;
private static final Byte LOCK = 0;
@Autowired
protected HtmlProcessorService htmlProcessorService;
@Autowired
protected XpathService xpathService;
@Autowired
private AppSettingsService appSettingsService;
@Autowired
private DataService dataService;
@Autowired
private ConnectionManager cm;
@Autowired
private DownloadSpeedService downloadSpeedService;
@Autowired
private PathService pathService;
@Autowired
private VipergirlsAuthService authService;
protected Host() {
}
abstract public String getHost();
abstract public String getLookup();
@@ -79,204 +20,7 @@ abstract public class Host {
return url.contains(getLookup());
}
public void download(final Post post, final Image image, final ImageFileData imageFileData) throws DownloadException, InterruptedException {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
try {
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
}
imageFileData.setPageUrl(image.getUrl());
/*
* HOST SPECIFIC
*/
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
setNameAndUrl(image.getUrl(), imageFileData, context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
log.debug(String.format("Building image request for %s", image.getUrl()));
setImageRequest(imageFileData);
/*
* END HOST SPECIFIC
*/
String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
log.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
imageFileData.setImageName(formatImageFileName);
HttpClient client = cm.getClient().build();
log.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest(), context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
File destinationFolder;
synchronized (LOCK) {
if (post.getPostFolderName() == null) {
pathService.createDefaultPostFolder(post);
}
destinationFolder = pathService.getDownloadDestinationFolder(post);
authService.leaveThanks(post);
}
File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
} finally {
if (image.getCurrent() == image.getTotal()) {
image.setStatus(Status.COMPLETE);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
}
File finalName = checkImageTypeAndRename(post, outputFile, imageFileData.getImageName(), image.getIndex());
imageFileData.setFileName(finalName.getName());
}
} catch (Exception e) {
if (Thread.interrupted() || imageFileData.getImageRequest().isAborted()) {
throw new InterruptedException("Download was interrupted");
}
throw new DownloadException(e);
}
}
private File checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
if (reader.getFormatName().toUpperCase().equals("JPEG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
imageName += ".jpg";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_jpg")) {
toReplace = "_jpg";
} else if (imageNameLC.endsWith("_jpeg")) {
toReplace = "_jpeg";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
}
}
} else if (reader.getFormatName().toUpperCase().equals("PNG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_png")) {
imageName += ".png";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_png")) {
toReplace = "_png";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
}
}
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
File downloadDestinationFolder = pathService.getDownloadDestinationFolder(post);
File outImage = new File(downloadDestinationFolder, (appSettingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
if (outImage.exists() && outImage.delete()) {
log.debug(String.format("%s is deleted", outImage.toString()));
}
return Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
}
}
final String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
final Response getResponse(final String url, final HttpClientContext context) throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
Header[] headers;
log.debug(String.format("Requesting %s", url));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
log.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
protected String appendUri(String uri, String appendQuery) throws URISyntaxException {
URI oldUri = new URI(uri);
String newQuery = oldUri.getQuery();
if (newQuery == null) {
newQuery = appendQuery;
} else {
newQuery += "&" + appendQuery;
}
return new URI(oldUri.getScheme(), oldUri.getAuthority(),
oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
}
private void setImageRequest(final ImageFileData imageFileData) {
HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
}
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException;
public abstract HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException;
@Override
public boolean equals(Object o) {
@@ -291,18 +35,6 @@ abstract public class Host {
return Objects.hash(getHost());
}
@Getter
public static class Response {
protected Response(Document document, Header[] headers) {
this.document = document;
this.headers = headers;
}
private Document document;
private Header[] headers;
}
@Override
public String toString() {
return getHost();
@@ -13,8 +13,10 @@ import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.io.IOException;
@@ -26,11 +28,17 @@ public class ImageBamHost extends Host {
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
private static final String IMG_XPATH = "//img[@class='image']";
@Autowired
private ConnectionManager cm;
private final HostService hostService;
private final ConnectionService cm;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
public ImageBamHost() {
super();
@Autowired
public ImageBamHost(HostService hostService, ConnectionService cm, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
this.hostService = hostService;
this.cm = cm;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
@@ -44,9 +52,9 @@ public class ImageBamHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Response response = getResponse(url, context);
HostService.Response response = hostService.getResponse(url, context);
Document doc = response.getDocument();
Node contDiv;
@@ -60,7 +68,7 @@ public class ImageBamHost extends Host {
if (contDiv != null) {
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
HttpGet httpGet = cm.buildHttpGet(url, context);
httpGet.addHeader("Referer", url);
log.debug(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
@@ -87,8 +95,7 @@ public class ImageBamHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@@ -18,8 +20,13 @@ public class ImageTwistHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "imagetwist.com";
public ImageTwistHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageTwistHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -33,9 +40,9 @@ public class ImageTwistHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
@@ -50,8 +57,7 @@ public class ImageTwistHost extends Host {
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -17,8 +19,13 @@ public class ImageVenueHost extends Host {
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
public ImageVenueHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageVenueHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -32,18 +39,18 @@ public class ImageVenueHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Response resp = getResponse(url, context);
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
//Button detected. No need to actually click it, just make the call again.
resp = getResponse(url, context);
resp = hostService.getResponse(url, context);
doc = resp.getDocument();
}
} catch (XpathException e) {
@@ -67,8 +74,7 @@ public class ImageVenueHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -17,8 +19,13 @@ public class ImageZillaHost extends Host {
private static final String lookup = "imagezilla.net/show";
private static final String IMG_XPATH = "//img[@id='photo']";
public ImageZillaHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageZillaHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -32,9 +39,9 @@ public class ImageZillaHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
String title;
try {
@@ -51,12 +58,11 @@ public class ImageZillaHost extends Host {
}
if (title == null || title.isEmpty()) {
title = getDefaultImageName(url);
title = hostService.getDefaultImageName(url);
}
try {
imageFileData.setImageUrl(url.replace("show", "images"));
imageFileData.setImageName(title);
return new HostService.NameUrl(title, url.replace("show", "images"));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -16,8 +18,13 @@ public class ImgSpiceHost extends Host {
private static final String host = "imgspice.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
public ImgSpiceHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -31,10 +38,10 @@ public class ImgSpiceHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Response resp = getResponse(url, context);
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
@@ -50,8 +57,7 @@ public class ImgSpiceHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -8,8 +8,8 @@ import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -18,11 +18,14 @@ public class ImgboxHost extends Host {
private static final String host = "imgbox.com";
private static final String IMG_XPATH = "//img[@id='img']";
@Autowired
private ConnectionManager cm;
private final HostService hostService;
private final XpathService xpathService;
public ImgboxHost() {
super();
@Autowired
public ImgboxHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -36,9 +39,9 @@ public class ImgboxHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
@@ -53,8 +56,7 @@ public class ImgboxHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -16,8 +16,10 @@ import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.io.IOException;
import java.util.ArrayList;
@@ -31,8 +33,18 @@ public class ImxHost extends Host {
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final HostService hostService;
private final XpathService xpathService;
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public ImxHost(HostService hostService, XpathService xpathService, ConnectionService cm, HtmlProcessorService htmlProcessorService) {
this.hostService = hostService;
this.xpathService = xpathService;
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -45,10 +57,10 @@ public class ImxHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Response resp = getResponse(url, context);
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node contDiv;
@@ -69,7 +81,7 @@ public class ImxHost extends Host {
}
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
HttpPost httpPost = cm.buildHttpPost(url, context);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", value));
try {
@@ -99,8 +111,7 @@ public class ImxHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -16,8 +18,14 @@ public class PimpandhostHost extends Host {
private static final String host = "pimpandhost.com";
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
public PimpandhostHost() {
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PimpandhostHost(HostService hostService, XpathService xpathService) {
super();
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -31,16 +39,16 @@ public class PimpandhostHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url;
try {
url = appendUri(_url.replace("http://", "https://"), "size=original");
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
} catch (Exception e) {
throw new HostException(e);
}
Response resp = getResponse(url, context);
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
@@ -56,8 +64,10 @@ public class PimpandhostHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle,
imgUrl
);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -16,6 +18,15 @@ public class PixRouteHost extends Host {
private static final String host = "pixroute.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixRouteHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
@@ -27,10 +38,10 @@ public class PixRouteHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
@@ -47,8 +58,10 @@ public class PixRouteHost extends Host {
try {
log.debug(String.format("Resolving name and image url for %s", url));
imageFileData.setImageUrl(imgNode.getAttributes().getNamedItem("src").getTextContent().trim());
imageFileData.setImageName(imgNode.getAttributes().getNamedItem("alt").getTextContent().trim());
return new HostService.NameUrl(
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
imgNode.getAttributes().getNamedItem("src").getTextContent().trim()
);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -17,8 +19,13 @@ public class PixhostHost extends Host {
private static final String lookup = "pixhost.to/show";
private static final String IMG_XPATH = "//img[@id='image']";
public PixhostHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixhostHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -32,9 +39,9 @@ public class PixhostHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
@@ -49,8 +56,7 @@ public class PixhostHost extends Host {
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1));
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -17,8 +19,13 @@ public class PixxxelsHost extends Host {
private static final String IMG_XPATH = "//*[@id='download']";
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
public PixxxelsHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixxxelsHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -32,10 +39,10 @@ public class PixxxelsHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Response resp = getResponse(url, context);
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode, titleNode;
@@ -55,8 +62,7 @@ public class PixxxelsHost extends Host {
String imgTitle = titleNode.getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@@ -19,8 +21,13 @@ public class PostImgHost extends Host {
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
private static final String IMG_XPATH = "//a[@id='download']";
public PostImgHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PostImgHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -34,10 +41,10 @@ public class PostImgHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
Node urlNode, titleNode;
try {
@@ -52,10 +59,9 @@ public class PostImgHost extends Host {
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> getDefaultImageName(url));
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> hostService.getDefaultImageName(url));
imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -2,12 +2,14 @@ package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
@@ -15,10 +17,15 @@ public class TurboImageHost extends Host {
private static final String host = "turboimagehost.com";
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
private static final String IMG_XPATH = "//img[@id='uImage']";
private static final String IMG_XPATH = "//img[@id='imageid']";
public TurboImageHost() {
super();
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public TurboImageHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
@@ -32,9 +39,9 @@ public class TurboImageHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = getResponse(url, context).getDocument();
Document doc = hostService.getResponse(url, context).getDocument();
String title;
try {
@@ -51,13 +58,12 @@ public class TurboImageHost extends Host {
}
if (title == null || title.isEmpty()) {
title = getDefaultImageName(url);
title = hostService.getDefaultImageName(url);
}
try {
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
imageFileData.setImageName(title);
return new HostService.NameUrl(title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,65 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class ViprImHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "vipr.im";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ViprImHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,13 +1,15 @@
package tn.mnlr.vripper.jpa;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.services.ThreadPoolService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -25,11 +27,26 @@ public class Management {
private final JdbcTemplate jdbcTemplate;
private final String backupFolder;
private final ThreadPoolService threadPoolService;
private final DownloadService downloadService;
@Autowired
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir) {
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName, ThreadPoolService threadPoolService, DownloadService downloadService) {
this.jdbcTemplate = jdbcTemplate;
this.backupFolder = baseDir + File.separator + ".vripper" + File.separator + "backup";
this.threadPoolService = threadPoolService;
this.downloadService = downloadService;
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
}
@PreDestroy
public void destroy() {
try {
threadPoolService.destroy();
downloadService.destroy();
this.jdbcTemplate.execute("SHUTDOWN");
} catch (Exception ignored) {
Thread.currentThread().interrupt();
}
}
@PostConstruct
@@ -14,8 +14,6 @@ public interface IImageRepository extends IRepository {
List<Image> findByPostId(String postId);
Integer countRemaining();
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
@@ -67,14 +67,6 @@ public class ImageRepository implements IImageRepository {
);
}
@Override
public Integer countRemaining() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.TOTAL = 0 OR image.TOTAL <> image.CURRENT",
Integer.class
);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
@@ -78,7 +78,7 @@ public class MetadataRepository implements IMetadataRepository {
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM METADATA WHERE POST_ID_REF = (SELECT post.ID FROM POST AS post INNER JOIN METADATA metadata ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?)",
"DELETE FROM METADATA AS metadata WHERE metadata.ID = (SELECT inner_metadata.ID FROM POST AS post INNER JOIN METADATA inner_metadata ON post.ID = inner_metadata.POST_ID_REF WHERE post.POST_ID = ?)",
postId
);
}
@@ -1,58 +0,0 @@
package tn.mnlr.vripper.q;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.function.CheckedRunnable;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
import java.util.Objects;
@Slf4j
public class DownloadJob implements CheckedRunnable {
private final DataService dataService;
@Getter
private Image image;
@Getter
private Post post;
@Getter
private final ImageFileData imageFileData = new ImageFileData();
DownloadJob(Post post, Image image) {
this.image = image;
this.post = post;
dataService = SpringContext.getBean(DataService.class);
}
@Override
public void run() throws Exception {
log.debug(String.format("Starting downloading %s", image.getUrl()));
image.getHost().download(post, image, imageFileData);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return Objects.equals(image, that.image) &&
Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image, post);
}
public void refresh() {
post = dataService.findPostById(post.getId()).orElseThrow();
image = dataService.findImageById(image.getId()).orElseThrow();
}
}
@@ -1,16 +0,0 @@
package tn.mnlr.vripper.q;
import lombok.Getter;
import lombok.Setter;
import org.apache.http.client.methods.HttpUriRequest;
@Getter
@Setter
public class ImageFileData {
private String pageUrl;
private String imageName;
private String fileName;
private String imageUrl;
private HttpUriRequest imageRequest;
}
@@ -2,8 +2,10 @@ package tn.mnlr.vripper.services;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
@@ -11,15 +13,17 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.download.DownloadJob;
import java.net.URI;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Service
@EnableScheduling
public class ConnectionManager {
public class ConnectionService {
private ConnectionManager() {
private ConnectionService() {
buildConnectionPool();
}
@@ -53,21 +57,33 @@ public class ConnectionManager {
.setDefaultRequestConfig(rc);
}
public HttpGet buildHttpGet(String url) {
public HttpGet buildHttpGet(String url, final HttpClientContext context) {
HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpGet);
return httpGet;
}
public HttpPost buildHttpPost(String url) {
public HttpPost buildHttpPost(String url, final HttpClientContext context) {
HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpPost);
return httpPost;
}
public HttpGet buildHttpGet(URI uri) {
public HttpGet buildHttpGet(URI uri, final HttpClientContext context) {
HttpGet httpGet = new HttpGet(uri);
httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpGet);
return httpGet;
}
public void addToContext(HttpClientContext context, AbstractExecutionAwareRequest request) {
if (context != null) {
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
requests.add(request);
}
}
}
}
@@ -30,7 +30,7 @@ public class DataService {
private final IImageRepository imageRepository;
private final IQueuedRepository queuedRepository;
private final IMetadataRepository metadataRepository;
private final AppSettingsService appSettingsService;
private final SettingsService settingsService;
private final PublishProcessor<Long> liveGrabQueue = PublishProcessor.create();
private final PublishProcessor<Long> liveImageUpdates = PublishProcessor.create();
@@ -60,12 +60,12 @@ public class DataService {
}
@Autowired
public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, AppSettingsService appSettingsService) {
public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, SettingsService settingsService) {
this.postRepository = postRepository;
this.imageRepository = imageRepository;
this.queuedRepository = queuedRepository;
this.metadataRepository = metadataRepository;
this.appSettingsService = appSettingsService;
this.settingsService = settingsService;
}
private void save(Post post) {
@@ -125,7 +125,7 @@ public class DataService {
} else {
post.setStatus(Status.COMPLETE);
updatePostStatus(post.getStatus(), post.getId());
if (appSettingsService.getSettings().getClearCompleted()) {
if (settingsService.getSettings().getClearCompleted()) {
remove(post.getPostId());
}
}
@@ -169,10 +169,6 @@ public class DataService {
}
public long countRemainingImages() {
return imageRepository.countRemaining();
}
public long countErrorImages() {
return imageRepository.countError();
}
@@ -6,15 +6,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.q.ExecutionService;
import tn.mnlr.vripper.q.PendingQ;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.download.PendingQueue;
import tn.mnlr.vripper.services.domain.GlobalState;
@Service
@EnableScheduling
public class GlobalStateService {
private final PendingQ pendingQ;
private final ExecutionService executionService;
private final PendingQueue pendingQueue;
private final DownloadService downloadService;
private final DataService dataService;
@Getter
@@ -24,18 +25,17 @@ public class GlobalStateService {
private final PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
@Autowired
public GlobalStateService(PendingQ pendingQ, ExecutionService executionService, DataService dataService) {
this.pendingQ = pendingQ;
this.executionService = executionService;
public GlobalStateService(PendingQueue pendingQueue, DownloadService downloadService, DataService dataService) {
this.pendingQueue = pendingQueue;
this.downloadService = downloadService;
this.dataService = dataService;
}
@Scheduled(fixedDelay = 3000)
private void interval() {
GlobalState newGlobalState = new GlobalState(
executionService.runningCount(),
pendingQ.size(),
dataService.countRemainingImages(),
downloadService.runningCount(),
pendingQueue.size(),
dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
@@ -0,0 +1,105 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@Service
@Slf4j
public class HostService {
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
@Autowired
public HostService(ConnectionService cm, HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
}
@Getter
public static class Response {
private final Document document;
private final Header[] headers;
protected Response(Document document, Header[] headers) {
this.document = document;
this.headers = headers;
}
}
@Getter
public static class NameUrl {
private String name;
private String url;
public NameUrl(String name, String url) {
this.name = name;
this.url = url;
}
}
public Response getResponse(final String url, final HttpClientContext context) throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url, context);
Header[] headers;
log.debug(String.format("Requesting %s", url));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
log.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
public String appendUri(String uri, String appendQuery) throws URISyntaxException {
URI oldUri = new URI(uri);
String newQuery = oldUri.getQuery();
if (newQuery == null) {
newQuery = appendQuery;
} else {
newQuery += "&" + appendQuery;
}
return new URI(oldUri.getScheme(), oldUri.getAuthority(),
oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
}
public String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.post;
package tn.mnlr.vripper.services;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -20,10 +20,6 @@ import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import tn.mnlr.vripper.services.XpathService;
import java.util.ArrayList;
import java.util.Arrays;
@@ -36,7 +32,7 @@ import java.util.stream.Collectors;
@Slf4j
@Service
public class MetadataCache {
public class MetadataService {
@Getter
static class Key {
@@ -67,15 +63,15 @@ public class MetadataCache {
private static final List<String> dictionary = Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
private final LoadingCache<Key, Metadata> cache;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
private final HtmlProcessorService htmlProcessorService;
private final XpathService xpathService;
@Autowired
public MetadataCache(ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService) {
public MetadataService(ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService) {
this.cm = cm;
this.vipergirlsAuthService = vipergirlsAuthService;
this.VGAuthService = VGAuthService;
this.htmlProcessorService = htmlProcessorService;
this.xpathService = xpathService;
@@ -99,7 +95,7 @@ public class MetadataCache {
}
private Metadata fetchMetadata(Key key) {
HttpGet httpGet = cm.buildHttpGet(key.getUrl());
HttpGet httpGet = cm.buildHttpGet(key.getUrl(), null);
Metadata metadata = new Metadata();
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> {
@@ -111,7 +107,7 @@ public class MetadataCache {
})
.run(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
@@ -1,27 +0,0 @@
package tn.mnlr.vripper.services;
import org.springframework.stereotype.Service;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
@Service
public class MutexService {
private final Map<String, ReentrantLock> postLock = new ConcurrentHashMap<>();
public synchronized void createPostLock(String postId) {
if (!postLock.containsKey(postId)) {
postLock.put(postId, new ReentrantLock());
}
}
public void removePostLock(String postId) {
postLock.remove(postId);
}
public ReentrantLock getPostLock(String postId) {
return postLock.get(postId);
}
}
@@ -13,90 +13,92 @@ import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.locks.ReentrantLock;
import java.util.Objects;
import java.util.stream.Collectors;
@Service
@Slf4j
public class PathService {
private final AppSettingsService appSettingsService;
public static final int MAX_ATTEMPTS = 24;
private final SettingsService settingsService;
private final DataService dataService;
private final MutexService mutexService;
private final CommonExecutor commonExecutor;
private final ThreadPoolService threadPoolService;
@Autowired
public PathService(AppSettingsService appSettingsService, DataService dataService, MutexService mutexService, CommonExecutor commonExecutor) {
this.appSettingsService = appSettingsService;
public PathService(SettingsService settingsService, DataService dataService, ThreadPoolService threadPoolService) {
this.settingsService = settingsService;
this.dataService = dataService;
this.mutexService = mutexService;
this.commonExecutor = commonExecutor;
this.threadPoolService = threadPoolService;
}
public final File getDownloadDestinationFolder(Post post) {
return _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), post.getPostFolderName());
return _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), post.getPostFolderName(), post.getPostId());
}
private File _getDownloadDestinationFolder(@NonNull String forum, @NonNull String threadTitle, @NonNull String title) {
File sourceFolder = appSettingsService.getSettings().getSubLocation() ? new File(appSettingsService.getSettings().getDownloadPath(), sanitize(forum)) : new File(appSettingsService.getSettings().getDownloadPath());
sourceFolder = appSettingsService.getSettings().getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
private File _getRootFolder(@NonNull String forum, @NonNull String threadTitle) {
File sourceFolder = settingsService.getSettings().getSubLocation() ? new File(settingsService.getSettings().getDownloadPath(), sanitize(forum)) : new File(settingsService.getSettings().getDownloadPath());
return settingsService.getSettings().getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
}
private File _getDownloadDestinationFolder(@NonNull String forum, @NonNull String threadTitle, @NonNull String title, @NonNull String postId) {
File sourceFolder = _getRootFolder(forum, threadTitle);
return new File(sourceFolder, title);
}
private File _createDownloadDestinationFolder(@NonNull String forum, @NonNull String threadTitle, @NonNull String title, @NonNull String postId) {
File sourceFolder = _getRootFolder(forum, threadTitle);
return new File(sourceFolder, settingsService.getSettings().getAppendPostId() ? title + "_" + postId : title);
}
public final void createDefaultPostFolder(Post post) {
File sourceFolder = _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(post.getTitle()));
File sourceFolder = _createDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(post.getTitle()), post.getPostId());
File destFolder = makeDirs(sourceFolder);
post.setPostFolderName(destFolder.getName());
dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
}
public final void rename(@NonNull String postId, @NonNull String altName) {
commonExecutor.getGeneralExecutor().submit(() -> {
Post post = dataService.findPostByPostId(postId).orElseThrow();
threadPoolService.getGeneralExecutor().submit(() -> {
if (altName.equals(post.getTitle())) {
return;
}
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
if (post.getPostFolderName() == null) {
return;
}
File newDestFolder = makeDirs(_getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(altName), postId));
File currentDesFolder = getDownloadDestinationFolder(post);
post.setPostFolderName(newDestFolder.getName());
dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
ReentrantLock postLock = mutexService.getPostLock(postId);
if (postLock != null) {
postLock.lock();
List<File> files = Arrays.stream(Objects.requireNonNull(currentDesFolder.listFiles())).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
for (File f : files) {
try {
Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
log.error(String.format("Failed to move files from %s to %s", currentDesFolder.toString(), newDestFolder.toString()), e);
return;
}
}
try {
Optional<Post> _post = dataService.findPostByPostId(postId);
if (_post.isEmpty()) {
return;
}
Post post = _post.get();
if (altName.equals(post.getTitle())) {
return;
}
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
if (post.getPostFolderName() == null) {
return;
}
File newDestFolder = makeDirs(_getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(altName)));
File currentDesFolder = getDownloadDestinationFolder(post);
post.setPostFolderName(newDestFolder.getName());
dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
List<File> files = Optional.ofNullable(currentDesFolder.listFiles()).stream().flatMap(Arrays::stream).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
for (File f : files) {
try {
Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
log.error(String.format("Failed to move files from %s to %s", currentDesFolder.toString(), newDestFolder.toString()), e);
return;
}
}
int attempts = 0;
while (currentDesFolder.exists() && attempts <= MAX_ATTEMPTS) {
attempts++;
if (!currentDesFolder.delete()) {
log.warn(String.format("Failed to remove %s", currentDesFolder.toString()));
}
} finally {
if (postLock != null) {
postLock.unlock();
try {
Thread.sleep(5_000);
} catch (InterruptedException ignored) {
}
}
if (attempts > MAX_ATTEMPTS) {
log.error(String.format("Failed to rename post %s", postId));
}
});
}
@@ -0,0 +1,156 @@
package tn.mnlr.vripper.services;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.download.PendingQueue;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.domain.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@Service
@Slf4j
public class PostService {
private final SettingsService settingsService;
private final PendingQueue pendingQueue;
private final DataService dataService;
private final ThreadPoolService threadPoolService;
private final Map<String, Future<?>> fetchingMetadata = new ConcurrentHashMap<>();
private final VGAuthService VGAuthService;
@Getter
private final LoadingCache<Queued, List<MultiPostItem>> cache;
@Autowired
public PostService(SettingsService settingsService, PendingQueue pendingQueue, DataService dataService, ThreadPoolService threadPoolService, VGAuthService VGAuthService) {
this.settingsService = settingsService;
this.pendingQueue = pendingQueue;
this.dataService = dataService;
this.threadPoolService = threadPoolService;
this.VGAuthService = VGAuthService;
CacheLoader<Queued, List<MultiPostItem>> loader = new CacheLoader<>() {
@Override
public List<MultiPostItem> load(@NonNull Queued multiPostItem) throws Exception {
ApiThreadParser apiThreadParser = new ApiThreadParser(multiPostItem);
return apiThreadParser.parse();
}
};
cache = CacheBuilder.newBuilder()
.expireAfterWrite(30, TimeUnit.MINUTES)
.build(loader);
}
public void addPost(String postId, String threadId) throws PostParseException {
if (dataService.exists(postId)) {
log.warn(String.format("skipping %s, already loaded", postId));
return;
}
ApiPostParser apiPostParser = new ApiPostParser(threadId, postId);
ApiPost apiPost = apiPostParser.parse();
if (apiPost.getPost().isEmpty()) {
throw new PostParseException(String.format("parsing failed for thread %s, post %s", threadId, postId));
}
Post post = apiPost.getPost().get();
Set<Image> images = apiPost.getImages();
dataService.newPost(post, images);
// Metadata thread
fetchingMetadata.put(post.getPostId(), threadPoolService.getGeneralExecutor().submit(new MetadataRunnable(post)));
if (settingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
try {
pendingQueue.enqueue(post, images);
} catch (InterruptedException e) {
log.warn("Interruption was caught");
Thread.currentThread().interrupt();
return;
}
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
log.debug("Auto start downloads option is disabled");
}
if (!settingsService.getSettings().getLeaveThanksOnStart()) {
VGAuthService.leaveThanks(post);
}
dataService.updatePostStatus(post.getStatus(), post.getId());
}
public void stopFetchingMetadata(Post post) {
this.fetchingMetadata.forEach((k, v) -> {
if (k.equals(post.getPostId())) {
v.cancel(true);
}
});
fetchingMetadata.remove(post.getPostId());
}
public void processMultiPost(List<Queued> queuedList) throws Exception {
for (Queued queued : queuedList) {
if (queued.getPostId() != null) {
addPost(queued.getPostId(), queued.getThreadId());
} else {
threadPoolService.getGeneralExecutor().submit(() -> this.multiPost(queued));
}
}
}
private void multiPost(Queued queued) {
List<MultiPostItem> multiPostItems;
try {
multiPostItems = cache.get(queued);
} catch (ExecutionException e) {
log.error(String.format("Failed to add post with thread id %s, postId %s", queued.getThreadId(), queued.getPostId()), e);
return;
}
queued.setTotal(multiPostItems.size());
queued.done();
log.debug(String.format("%d found for %s", multiPostItems.size(), queued.getLink()));
if (multiPostItems.size() == 1) {
try {
addPost(multiPostItems.get(0).getPostId(), multiPostItems.get(0).getThreadId());
} catch (PostParseException e) {
log.error(String.format("Failed to add post with postId %s", multiPostItems.get(0).getPostId()), e);
return;
}
log.debug(String.format("threadId %s, postId %s is added automatically for download", queued.getThreadId(), queued.getPostId()));
} else {
if(dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) {
dataService.newQueueLink(queued);
} else {
log.info(String.format("Thread with id = %s is already loaded", queued.getThreadId()));
}
}
}
public void remove(String threadId) {
dataService.removeQueueLink(threadId);
}
}
@@ -28,19 +28,15 @@ import static java.nio.file.StandardOpenOption.*;
@Getter
@Setter
@Slf4j
public class AppSettingsService {
private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
private final String baseDir;
public class SettingsService {
private final Path configPath;
private final ObjectMapper om = new ObjectMapper();
private Settings settings = new Settings();
public AppSettingsService(@Value("${base.dir}") String baseDir) {
this.baseDir = baseDir;
this.configPath = Paths.get(baseDir, ".vripper", "config.json");
public SettingsService(@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
@@ -51,14 +47,15 @@ public class AppSettingsService {
public void newSettings(Settings settings) {
if (settings.getVLogin()) {
if (settings.getVLogin() != null && settings.getVLogin()) {
if (!this.settings.getVPassword().equals(settings.getVPassword())) {
settings.setVPassword(settings.getVPassword());
settings.setVPassword(DigestUtils.md5Hex(settings.getVPassword()));
}
} else {
settings.setVUsername("");
settings.setVPassword("");
settings.setVThanks(false);
settings.setVLogin(false);
}
this.settings = settings;
@@ -80,7 +77,7 @@ public class AppSettingsService {
}
if (settings.getDownloadPath() == null) {
settings.setDownloadPath(baseDir);
settings.setDownloadPath(System.getProperty("user.home"));
}
if (settings.getMaxThreads() == null) {
@@ -135,23 +132,20 @@ public class AppSettingsService {
settings.setDarkTheme(false);
}
if (settings.getViewPhotos() == null) {
settings.setViewPhotos(false);
if (settings.getAppendPostId() == null) {
settings.setAppendPostId(true);
}
if (settings.getNotification() == null) {
settings.setNotification(false);
if (settings.getLeaveThanksOnStart() == null) {
settings.setLeaveThanksOnStart(false);
}
save();
}
@PreDestroy
public void save() {
try {
// force disable gallery
settings.setViewPhotos(false);
Files.write(configPath, om.writeValueAsBytes(settings), CREATE, WRITE, TRUNCATE_EXISTING, SYNC);
} catch (IOException e) {
@@ -211,43 +205,50 @@ public class AppSettingsService {
@JsonProperty("downloadPath")
private String downloadPath;
@JsonProperty("maxThreads")
private Integer maxThreads;
@JsonProperty("maxTotalThreads")
private Integer maxTotalThreads;
@JsonProperty("autoStart")
private Boolean autoStart;
@JsonProperty("vLogin")
private Boolean vLogin;
@JsonProperty("vUsername")
private String vUsername;
@JsonProperty("vPassword")
private String vPassword;
@JsonProperty("vThanks")
private Boolean vThanks;
@JsonProperty("desktopClipboard")
private Boolean desktopClipboard;
@JsonProperty("forceOrder")
private Boolean forceOrder;
@JsonProperty("subLocation")
private Boolean subLocation;
@JsonProperty("threadSubLocation")
private Boolean threadSubLocation;
@JsonProperty("clearCompleted")
private Boolean clearCompleted;
@JsonProperty("viewPhotos")
private Boolean viewPhotos;
@JsonProperty("notification")
private Boolean notification;
@JsonProperty("darkTheme")
private Boolean darkTheme;
public void setVPassword(String vPassword) {
if (vPassword == null || vPassword.isEmpty()) {
this.vPassword = "";
} else {
this.vPassword = DigestUtils.md5Hex(vPassword);
}
}
@JsonProperty("appendPostId")
private Boolean appendPostId;
@JsonProperty("leaveThanksOnStart")
private Boolean leaveThanksOnStart;
}
}
@@ -11,7 +11,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@Service
public class CommonExecutor {
public class ThreadPoolService {
@Getter
private final ExecutorService generalExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
private final ExecutorService schedulerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
@@ -19,11 +19,10 @@ public class CommonExecutor {
@Getter
private final Scheduler scheduler = Schedulers.from(schedulerExecutor);
@PreDestroy
private void destroy() throws InterruptedException {
public void destroy() throws Exception {
generalExecutor.shutdown();
generalExecutor.awaitTermination(10, TimeUnit.SECONDS);
generalExecutor.awaitTermination(5, TimeUnit.SECONDS);
schedulerExecutor.shutdown();
schedulerExecutor.awaitTermination(10, TimeUnit.SECONDS);
schedulerExecutor.awaitTermination(5, TimeUnit.SECONDS);
}
}
@@ -24,11 +24,11 @@ import java.util.List;
@Service
@Slf4j
public class VipergirlsAuthService {
public class VGAuthService {
private final ConnectionManager cm;
private final AppSettingsService appSettingsService;
private final CommonExecutor commonExecutor;
private final ConnectionService cm;
private final SettingsService settingsService;
private final ThreadPoolService threadPoolService;
private final DataService dataService;
@Getter
@@ -44,10 +44,10 @@ public class VipergirlsAuthService {
private final PublishProcessor<String> loggedInUser = PublishProcessor.create();
@Autowired
public VipergirlsAuthService(ConnectionManager cm, AppSettingsService appSettingsService, CommonExecutor commonExecutor, DataService dataService) {
public VGAuthService(ConnectionService cm, SettingsService settingsService, ThreadPoolService threadPoolService, DataService dataService) {
this.cm = cm;
this.appSettingsService = appSettingsService;
this.commonExecutor = commonExecutor;
this.settingsService = settingsService;
this.threadPoolService = threadPoolService;
this.dataService = dataService;
}
@@ -66,7 +66,7 @@ public class VipergirlsAuthService {
log.info("Authenticating using ViperGirls credentials");
authenticated = false;
if (!appSettingsService.getSettings().getVLogin()) {
if (!settingsService.getSettings().getVLogin()) {
log.debug("Authentication option is disabled");
context.getCookieStore().clear();
loggedUser = "";
@@ -74,8 +74,8 @@ public class VipergirlsAuthService {
return;
}
String username = appSettingsService.getSettings().getVUsername();
String password = appSettingsService.getSettings().getVPassword();
String username = settingsService.getSettings().getVUsername();
String password = settingsService.getSettings().getVPassword();
if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
@@ -85,7 +85,7 @@ public class VipergirlsAuthService {
return;
}
HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login");
HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login", null);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("vb_login_username", username));
@@ -134,11 +134,11 @@ public class VipergirlsAuthService {
}
public void leaveThanks(Post post) {
if (!appSettingsService.getSettings().getVLogin()) {
if (!settingsService.getSettings().getVLogin()) {
log.debug("Authentication with ViperGirls option is disabled");
return;
}
if (!appSettingsService.getSettings().getVThanks()) {
if (!settingsService.getSettings().getVThanks()) {
log.debug("Leave thanks option is disabled");
return;
}
@@ -150,7 +150,7 @@ public class VipergirlsAuthService {
log.debug("Already left a thanks");
return;
}
commonExecutor.getGeneralExecutor().submit(() -> {
threadPoolService.getGeneralExecutor().submit(() -> {
try {
postThanks(post);
} catch (Exception e) {
@@ -161,7 +161,7 @@ public class VipergirlsAuthService {
private void postThanks(Post post) throws VripperException {
HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php");
HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php", null);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("do", "post_thanks_add"));
params.add(new BasicNameValuePair("using_ajax", "1"));
@@ -1,86 +0,0 @@
package tn.mnlr.vripper.services;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.post.CachedPost;
import tn.mnlr.vripper.services.post.PostService;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@Service
@Slf4j
public class VGHandler {
private final DataService dataService;
private final PostService postService;
private final CommonExecutor commonExecutor;
@Getter
private final LoadingCache<Queued, List<CachedPost>> cache;
@Autowired
public VGHandler(DataService dataService, PostService postService, CommonExecutor commonExecutor) {
this.dataService = dataService;
this.postService = postService;
this.commonExecutor = commonExecutor;
CacheLoader<Queued, List<CachedPost>> loader = new CacheLoader<>() {
@Override
public List<CachedPost> load(@NonNull Queued queuedVGLink) throws Exception {
VRThreadParser vrThreadParser = new VRThreadParser(queuedVGLink);
return vrThreadParser.parse();
}
};
cache = CacheBuilder.newBuilder()
.expireAfterWrite(30, TimeUnit.MINUTES)
.build(loader);
}
public void handle(List<Queued> queuedList) throws Exception {
for (Queued queued : queuedList) {
if (queued.getPostId() != null) {
postService.addPost(queued.getPostId(), queued.getThreadId());
} else {
Runnable runnable = () -> {
List<CachedPost> cachedPosts;
try {
cachedPosts = cache.get(queued);
} catch (ExecutionException e) {
log.error(String.format("Failed to add post with thread id %s, postId %s", queued.getThreadId(), queued.getPostId()), e);
return;
}
queued.setTotal(cachedPosts.size());
queued.done();
log.debug(String.format("%d found for %s", cachedPosts.size(), queued.getLink()));
if (cachedPosts.size() == 1) {
try {
postService.addPost(cachedPosts.get(0).getPostId(), cachedPosts.get(0).getThreadId());
} catch (PostParseException e) {
log.error(String.format("Failed to add post with postId %s", cachedPosts.get(0).getPostId()), e);
return;
}
log.debug(String.format("threadId %s, postId %s is added automatically for download", queued.getThreadId(), queued.getPostId()));
} else {
dataService.newQueueLink(queued);
}
};
commonExecutor.getGeneralExecutor().submit(runnable);
}
}
}
public void remove(String threadId) {
dataService.removeQueueLink(threadId);
}
}
@@ -1,163 +0,0 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.post.CachedPost;
import javax.xml.parsers.SAXParserFactory;
import java.io.BufferedInputStream;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@Slf4j
public class VRThreadParser {
private static final String VR_API = "https://vipergirls.to/vr.php";
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final Queued queued;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
VRThreadParser(Queued queued) {
this.queued = queued;
this.cm = SpringContext.getBean(ConnectionManager.class);
this.vipergirlsAuthService = SpringContext.getBean(VipergirlsAuthService.class);
}
public List<CachedPost> parse() throws PostParseException {
log.debug(String.format("Parsing thread %s", queued));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(VR_API);
uriBuilder.setParameter("t", queued.getThreadId());
httpGet = cm.buildHttpGet(uriBuilder.build());
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
VRThreadHandler handler = new VRThreadHandler(queued);
AtomicReference<Throwable> thr = new AtomicReference<>();
log.debug(String.format("Requesting %s", httpGet));
List<CachedPost> posts = Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
try {
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
return handler.getPosts();
} catch (Exception e) {
throw new PostParseException(String.format("Failed to parse thread %s", queued), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
if (thr.get() != null) {
log.error(String.format("parsing failed for thread %s", queued), thr.get());
throw new PostParseException(thr.get());
}
return posts;
}
}
class VRThreadHandler extends DefaultHandler {
private final Queued queued;
private final Collection<Host> supportedHosts;
private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
private List<String> previews = new ArrayList<>();
private String threadTitle;
private String postId;
private String postTitle;
private int imageCount;
private int postCounter;
private int previewCounter = 0;
@Getter
private List<CachedPost> posts = new ArrayList<>();
VRThreadHandler(Queued queued) {
this.queued = queued;
this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
}
@Override
public void startDocument() {
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
postId = attributes.getValue("id").trim();
postCounter = Integer.parseInt(attributes.getValue("number").trim());
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
break;
case "image":
Optional.ofNullable(attributes.getValue("main_url"))
.map(String::trim)
.flatMap(mainUrl -> supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
.ifPresent(host -> Optional.ofNullable(hostMap.get(host)).ifPresentOrElse(AtomicInteger::incrementAndGet, () -> hostMap.put(host, new AtomicInteger(0))));
if (previewCounter++ < 4) {
Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equals(qName.toLowerCase())) {
if (imageCount != 0) {
posts.add(new CachedPost(
queued.getThreadId(),
postId,
postCounter,
postTitle,
imageCount,
String.format("https://vipergirls.to/threads/?p=%s&viewfull=1#post%s", postId, postId),
previews,
hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
));
queued.increment();
}
previewCounter = 0;
previews = new ArrayList<>();
hostMap.clear();
}
}
@Override
public void endDocument() {
queued.done();
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.post;
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
import tn.mnlr.vripper.jpa.domain.Image;
@@ -7,14 +7,14 @@ import tn.mnlr.vripper.jpa.domain.Post;
import java.util.Optional;
import java.util.Set;
public class ParseResult {
public class ApiPost {
private final Post post;
@Getter
private final Set<Image> images;
ParseResult(Post post, Set<Image> images) {
public ApiPost(Post post, Set<Image> images) {
this.post = post;
this.images = images;
}
@@ -22,4 +22,4 @@ public class ParseResult {
public Optional<Post> getPost() {
return Optional.ofNullable(post);
}
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.post;
package tn.mnlr.vripper.services.domain;
import lombok.extern.slf4j.Slf4j;
import org.xml.sax.Attributes;
@@ -7,6 +7,7 @@ import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.domain.ApiPost;
import java.util.Collection;
import java.util.HashSet;
@@ -15,7 +16,7 @@ import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
class VRApiPostHandler extends DefaultHandler {
class ApiPostHandler extends DefaultHandler {
private final Collection<Host> supportedHosts;
@@ -32,7 +33,7 @@ class VRApiPostHandler extends DefaultHandler {
private Post parsedPost;
VRApiPostHandler(String threadId, String postId) {
ApiPostHandler(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
this.postUrl = String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", this.threadId, this.postId, this.postId);
@@ -40,8 +41,8 @@ class VRApiPostHandler extends DefaultHandler {
}
public ParseResult getParsedPost() {
return new ParseResult(parsedPost, images);
public ApiPost getParsedPost() {
return new ApiPost(parsedPost, images);
}
@Override
@@ -105,4 +106,4 @@ class VRApiPostHandler extends DefaultHandler {
previews = new HashSet<>();
}
}
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services.post;
package tn.mnlr.vripper.services.domain;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
@@ -11,15 +11,15 @@ import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.VGAuthService;
import javax.xml.parsers.SAXParserFactory;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public class VRPostParser {
public class ApiPostParser {
private static final String VR_API = "https://vipergirls.to/vr.php";
@@ -27,32 +27,32 @@ public class VRPostParser {
private final String threadId;
private final String postId;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
public VRPostParser(String threadId, String postId) {
public ApiPostParser(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
this.cm = SpringContext.getBean(ConnectionManager.class);
this.vipergirlsAuthService = SpringContext.getBean(VipergirlsAuthService.class);
this.cm = SpringContext.getBean(ConnectionService.class);
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
}
public ParseResult parse() throws PostParseException {
public ApiPost parse() throws PostParseException {
log.debug(String.format("Parsing post %s", postId));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(VR_API);
uriBuilder.setParameter("p", postId);
httpGet = cm.buildHttpGet(uriBuilder.build());
httpGet = cm.buildHttpGet(uriBuilder.build(), null);
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
AtomicReference<Throwable> thr = new AtomicReference<>();
VRApiPostHandler handler = new VRApiPostHandler(threadId, postId);
ApiPostHandler apiPostHandler = new ApiPostHandler(threadId, postId);
log.debug(String.format("Requesting %s", httpGet));
ParseResult post = getPost(httpGet, handler, thr);
ApiPost post = getPost(httpGet, apiPostHandler, thr);
if (thr.get() != null) {
log.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
throw new PostParseException(thr.get());
@@ -60,19 +60,19 @@ public class VRPostParser {
return post;
}
private ParseResult getPost(HttpGet httpGet, VRApiPostHandler handler, AtomicReference<Throwable> thr) {
private ApiPost getPost(HttpGet httpGet, ApiPostHandler apiPostHandler, AtomicReference<Throwable> thr) {
return Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
factory.newSAXParser().parse(response.getEntity().getContent(), handler);
factory.newSAXParser().parse(response.getEntity().getContent(), apiPostHandler);
EntityUtils.consumeQuietly(response.getEntity());
return handler.getParsedPost();
return apiPostHandler.getParsedPost();
}
});
}
@@ -0,0 +1,90 @@
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
public class ApiThreadHandler extends DefaultHandler {
private final Queued queued;
private final Collection<Host> supportedHosts;
private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
private List<String> previews = new ArrayList<>();
private String threadTitle;
private String postId;
private String postTitle;
private int imageCount;
private int postCounter;
private int previewCounter = 0;
@Getter
private final List<MultiPostItem> posts = new ArrayList<>();
public ApiThreadHandler(Queued queued) {
this.queued = queued;
this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
}
@Override
public void startDocument() {
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
postId = attributes.getValue("id").trim();
postCounter = Integer.parseInt(attributes.getValue("number").trim());
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
break;
case "image":
Optional.ofNullable(attributes.getValue("main_url"))
.map(String::trim)
.flatMap(mainUrl -> supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
.ifPresent(host -> Optional.ofNullable(hostMap.get(host)).ifPresentOrElse(AtomicInteger::incrementAndGet, () -> hostMap.put(host, new AtomicInteger(1))));
if (previewCounter++ < 4) {
Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equals(qName.toLowerCase())) {
if (imageCount != 0) {
posts.add(new MultiPostItem(
queued.getThreadId(),
postId,
postCounter,
postTitle,
imageCount,
String.format("https://vipergirls.to/threads/?p=%s&viewfull=1#post%s", postId, postId),
previews,
hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
));
queued.increment();
}
previewCounter = 0;
previews = new ArrayList<>();
hostMap.clear();
}
}
@Override
public void endDocument() {
queued.done();
}
}
@@ -0,0 +1,80 @@
package tn.mnlr.vripper.services.domain;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.VGAuthService;
import javax.xml.parsers.SAXParserFactory;
import java.io.BufferedInputStream;
import java.net.URISyntaxException;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public class ApiThreadParser {
private static final String VR_API = "https://vipergirls.to/vr.php";
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final Queued queued;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
public ApiThreadParser(Queued queued) {
this.queued = queued;
this.cm = SpringContext.getBean(ConnectionService.class);
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
}
public List<MultiPostItem> parse() throws PostParseException {
log.debug(String.format("Parsing thread %s", queued));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(VR_API);
uriBuilder.setParameter("t", queued.getThreadId());
httpGet = cm.buildHttpGet(uriBuilder.build(), null);
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
ApiThreadHandler apiThreadHandler = new ApiThreadHandler(queued);
AtomicReference<Throwable> thr = new AtomicReference<>();
log.debug(String.format("Requesting %s", httpGet));
List<MultiPostItem> posts = Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
try {
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), apiThreadHandler);
return apiThreadHandler.getPosts();
} catch (Exception e) {
throw new PostParseException(String.format("Failed to parse thread %s", queued), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
if (thr.get() != null) {
log.error(String.format("parsing failed for thread %s", queued), thr.get());
throw new PostParseException(thr.get());
}
return posts;
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services;
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.services;
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
@@ -8,13 +8,11 @@ import java.util.Objects;
public class GlobalState {
private final long running;
private final long queued;
private final long remaining;
private final long error;
GlobalState(long running, long queued, long remaining, long error) {
public GlobalState(long running, long remaining, long error) {
this.running = running;
this.queued = queued;
this.remaining = remaining;
this.error = error;
}
@@ -24,11 +22,11 @@ public class GlobalState {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GlobalState that = (GlobalState) o;
return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
return running == that.running && remaining == that.remaining && error == that.error;
}
@Override
public int hashCode() {
return Objects.hash(running, queued, remaining, error);
return Objects.hash(running, remaining, error);
}
}
@@ -1,6 +1,7 @@
package tn.mnlr.vripper.services.post;
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.FailsafeException;
import org.apache.http.impl.execchain.RequestAbortedException;
@@ -8,6 +9,7 @@ import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.MetadataService;
@Slf4j
public class MetadataRunnable implements Runnable {
@@ -15,19 +17,23 @@ public class MetadataRunnable implements Runnable {
@Getter
private final Post post;
private final MetadataCache metadataCache;
private final MetadataService metadataService;
private final DataService dataService;
public MetadataRunnable(Post post) {
public MetadataRunnable(@NonNull Post post) {
this.post = post;
this.metadataCache = SpringContext.getBean(MetadataCache.class);
this.metadataService = SpringContext.getBean(MetadataService.class);
this.dataService = SpringContext.getBean(DataService.class);
}
@Override
public void run() {
try {
Metadata metadata = metadataCache.get(post);
if(Thread.interrupted()) {
log.debug(String.format("Metadata fetching for postId=%s, threadId=%s interrupted", post.getPostId(), post.getThreadId()));
return;
}
Metadata metadata = metadataService.get(post);
dataService.setMetadata(post, metadata);
} catch (Exception e) {
if (e.getCause() instanceof InterruptedException || (e.getCause() instanceof FailsafeException && (e.getCause().getCause() instanceof InterruptedException || e.getCause().getCause() instanceof RequestAbortedException))) {
@@ -1,12 +1,11 @@
package tn.mnlr.vripper.services.post;
package tn.mnlr.vripper.services.domain;
import lombok.Getter;
import java.util.List;
@Getter
public class CachedPost {
private final String type = "postParse";
public class MultiPostItem {
private final String threadId;
private final String postId;
private final int number;
@@ -16,7 +15,7 @@ public class CachedPost {
private final List<String> previews;
private final String hosts;
public CachedPost(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
public MultiPostItem(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
this.threadId = threadId;
this.postId = postId;
this.number = number;
@@ -1,85 +0,0 @@
package tn.mnlr.vripper.services.post;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.q.PendingQ;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.CommonExecutor;
import tn.mnlr.vripper.services.DataService;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
@Service
@Slf4j
public class PostService {
private final AppSettingsService appSettingsService;
private final PendingQ pendingQ;
private final DataService dataService;
private final CommonExecutor commonExecutor;
private final Map<String, Future<?>> fetchingMetadata = new ConcurrentHashMap<>();
@Autowired
public PostService(AppSettingsService appSettingsService, PendingQ pendingQ, DataService dataService, DataService dataService1, CommonExecutor commonExecutor) {
this.appSettingsService = appSettingsService;
this.pendingQ = pendingQ;
this.dataService = dataService1;
this.commonExecutor = commonExecutor;
}
public void addPost(String postId, String threadId) throws PostParseException {
if (dataService.exists(postId)) {
log.warn(String.format("skipping %s, already loaded", postId));
return;
}
VRPostParser vrPostParser = new VRPostParser(threadId, postId);
ParseResult parseResult = vrPostParser.parse();
if (parseResult.getPost().isEmpty()) {
throw new PostParseException(String.format("parsing failed for thread %s, post %s", threadId, postId));
}
Post post = parseResult.getPost().get();
Set<Image> images = parseResult.getImages();
dataService.newPost(post, images);
// Metadata thread
fetchingMetadata.put(post.getPostId(), commonExecutor.getGeneralExecutor().submit(new MetadataRunnable(post)));
if (appSettingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
try {
pendingQ.enqueue(post, images);
} catch (InterruptedException e) {
log.warn("Interruption was caught");
Thread.currentThread().interrupt();
return;
}
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
log.debug("Auto start downloads option is disabled");
}
dataService.updatePostStatus(post.getStatus(), post.getId());
}
public void stopFetchingMetadata(Post post) {
this.fetchingMetadata.forEach((k, v) -> {
if (k.equals(post.getPostId())) {
v.cancel(true);
}
});
fetchingMetadata.remove(post.getPostId());
}
}
@@ -5,18 +5,18 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.q.ExecutionService;
import tn.mnlr.vripper.services.CommonExecutor;
import tn.mnlr.vripper.services.ThreadPoolService;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PathService;
import tn.mnlr.vripper.services.VGHandler;
import tn.mnlr.vripper.services.post.CachedPost;
import tn.mnlr.vripper.services.post.PostService;
import tn.mnlr.vripper.web.restendpoints.domain.posts.*;
import tn.mnlr.vripper.services.domain.MultiPostItem;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.web.restendpoints.domain.*;
import tn.mnlr.vripper.web.restendpoints.domain.PostId;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
import tn.mnlr.vripper.web.restendpoints.exceptions.ServerErrorException;
@@ -39,21 +39,19 @@ public class PostRestEndpoint {
private final DataService dataService;
private final PathService pathService;
private final VGHandler vgHandler;
private final ExecutionService executionService;
private final DownloadService downloadService;
private final PostService postService;
private final CommonExecutor commonExecutor;
private final ThreadPoolService threadPoolService;
private static final Byte LOCK = -1;
private static final Object LOCK = new Object();
@Autowired
public PostRestEndpoint(DataService dataService, PathService pathService, VGHandler vgHandler, ExecutionService executionService, PostService postService, CommonExecutor commonExecutor) {
public PostRestEndpoint(DataService dataService, PathService pathService, DownloadService downloadService, PostService postService, ThreadPoolService threadPoolService) {
this.dataService = dataService;
this.pathService = pathService;
this.vgHandler = vgHandler;
this.executionService = executionService;
this.downloadService = downloadService;
this.postService = postService;
this.commonExecutor = commonExecutor;
this.threadPoolService = threadPoolService;
}
@PostMapping("/post")
@@ -84,7 +82,7 @@ public class PostRestEndpoint {
queuedList.add(new Queued(url, threadId, postId));
}
try {
vgHandler.handle(queuedList);
postService.processMultiPost(queuedList);
} catch (Exception e) {
log.error("Failed to parse links", e);
throw new ServerErrorException(e.getMessage());
@@ -96,7 +94,7 @@ public class PostRestEndpoint {
@ResponseStatus(value = HttpStatus.OK)
public void restartPost(@RequestBody @NonNull List<PostId> postIds) {
synchronized (LOCK) {
executionService.restartAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
downloadService.restartAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
}
}
@@ -105,7 +103,7 @@ public class PostRestEndpoint {
public void addPost(@RequestBody List<PostToAdd> posts) {
synchronized (LOCK) {
for (PostToAdd post : posts) {
commonExecutor.getGeneralExecutor().submit(() -> {
threadPoolService.getGeneralExecutor().submit(() -> {
try {
postService.addPost(post.getPostId(), post.getThreadId());
} catch (PostParseException e) {
@@ -145,7 +143,7 @@ public class PostRestEndpoint {
@ResponseStatus(value = HttpStatus.OK)
public void restartPost() {
synchronized (LOCK) {
executionService.restartAll(null);
downloadService.restartAll(null);
}
}
@@ -153,7 +151,7 @@ public class PostRestEndpoint {
@ResponseStatus(value = HttpStatus.OK)
public void stop(@RequestBody @NonNull List<PostId> postIds) {
synchronized (LOCK) {
executionService.stopAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
downloadService.stopAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
}
}
@@ -161,7 +159,7 @@ public class PostRestEndpoint {
@ResponseStatus(value = HttpStatus.OK)
public void stopAll() {
synchronized (LOCK) {
executionService.stopAll(null);
downloadService.stopAll(null);
}
}
@@ -171,7 +169,7 @@ public class PostRestEndpoint {
synchronized (LOCK) {
List<RemoveResult> result = new ArrayList<>();
List<String> collect = postIds.stream().map(PostId::getPostId).peek(e -> result.add(new RemoveResult(e))).collect(Collectors.toList());
executionService.stopAll(collect);
downloadService.stopAll(collect);
dataService.removeAll(collect);
return result;
}
@@ -239,10 +237,10 @@ public class PostRestEndpoint {
@GetMapping("/grab/{threadId}")
@ResponseStatus(value = HttpStatus.OK)
public List<CachedPost> grab(@PathVariable("threadId") @NonNull String threadId) {
public List<MultiPostItem> grab(@PathVariable("threadId") @NonNull String threadId) {
Queued queued = dataService.findQueuedByThreadId(threadId).orElseThrow(() -> new NotFoundException(String.format("Unable to find links for threadId = %s", threadId)));
try {
return vgHandler.getCache().get(queued);
return postService.getCache().get(queued);
} catch (ExecutionException e) {
log.error(String.format("Failed to get links for threadId = %s", threadId), e);
throw new ServerErrorException(String.format("Failed to get links for threadId = %s", threadId));
@@ -252,7 +250,7 @@ public class PostRestEndpoint {
@PostMapping("/grab/remove")
@ResponseStatus(value = HttpStatus.OK)
public ThreadId grabRemove(@RequestBody @NonNull ThreadId threadId) {
vgHandler.remove(threadId.getThreadId());
postService.remove(threadId.getThreadId());
return threadId;
}
}
@@ -5,8 +5,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import tn.mnlr.vripper.exception.ValidationException;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import tn.mnlr.vripper.services.SettingsService;
import tn.mnlr.vripper.services.VGAuthService;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
@RestController
@@ -14,48 +14,48 @@ import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
@CrossOrigin(value = "*")
public class SettingsRestEndpoint {
private final AppSettingsService appSettingsService;
private final VipergirlsAuthService vipergirlsAuthService;
private final SettingsService settingsService;
private final VGAuthService VGAuthService;
@Autowired
public SettingsRestEndpoint(AppSettingsService appSettingsService, VipergirlsAuthService vipergirlsAuthService) {
this.appSettingsService = appSettingsService;
this.vipergirlsAuthService = vipergirlsAuthService;
public SettingsRestEndpoint(SettingsService settingsService, VGAuthService VGAuthService) {
this.settingsService = settingsService;
this.VGAuthService = VGAuthService;
}
@PostMapping("/settings/theme")
@ResponseStatus(value = HttpStatus.OK)
public AppSettingsService.Theme postTheme(@RequestBody AppSettingsService.Theme theme) {
this.appSettingsService.setTheme(theme);
return appSettingsService.getTheme();
public SettingsService.Theme postTheme(@RequestBody SettingsService.Theme theme) {
this.settingsService.setTheme(theme);
return settingsService.getTheme();
}
@GetMapping("/settings/theme")
@ResponseStatus(value = HttpStatus.OK)
public AppSettingsService.Theme getTheme() {
return appSettingsService.getTheme();
public SettingsService.Theme getTheme() {
return settingsService.getTheme();
}
@PostMapping("/settings")
@ResponseStatus(value = HttpStatus.OK)
public AppSettingsService.Settings postSettings(@RequestBody AppSettingsService.Settings settings) throws Exception {
public SettingsService.Settings postSettings(@RequestBody SettingsService.Settings settings) throws Exception {
try {
this.appSettingsService.check(settings);
this.settingsService.check(settings);
} catch (ValidationException e) {
log.error("Invalid settings", e);
throw new BadRequestException(e.getMessage());
}
this.appSettingsService.newSettings(settings);
vipergirlsAuthService.authenticate();
this.settingsService.newSettings(settings);
VGAuthService.authenticate();
return getAppSettingsService();
}
@GetMapping("/settings")
@ResponseStatus(value = HttpStatus.OK)
public AppSettingsService.Settings getAppSettingsService() {
public SettingsService.Settings getAppSettingsService() {
return appSettingsService.getSettings();
return settingsService.getSettings();
}
}
@@ -1,18 +0,0 @@
package tn.mnlr.vripper.web.restendpoints;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.security.Principal;
@RestController
@CrossOrigin(value = "*")
public class UserRestEndpoint {
@RequestMapping("/user")
public Principal user(Principal user) {
return user;
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web;
package tn.mnlr.vripper.web.restendpoints;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
@@ -0,0 +1,12 @@
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.*;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class AltPostName {
private String postId;
private String altName;
}
@@ -1,9 +1,6 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import lombok.*;
@Getter
@Setter
@@ -15,4 +12,4 @@ public class DownloadPath {
public DownloadPath(String path) {
this.path = path;
}
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -15,4 +15,4 @@ public class PostId {
public PostId(String postId) {
this.postId = postId;
}
}
}
@@ -0,0 +1,12 @@
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.*;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class PostToAdd {
private String threadId;
private String postId;
}
@@ -1,9 +1,6 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import lombok.*;
import java.util.List;
@@ -19,4 +16,4 @@ public class RemoveAllResult {
this.removed = postIds.size();
this.postIds = postIds;
}
}
}
@@ -1,9 +1,6 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import lombok.*;
@Getter
@Setter
@@ -0,0 +1,11 @@
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.*;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class ThreadId {
private String threadId;
}
@@ -0,0 +1,11 @@
package tn.mnlr.vripper.web.restendpoints.domain;
import lombok.*;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class ThreadUrl {
private String url;
}
@@ -1,15 +0,0 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class AltPostName {
private String postId;
private String altName;
}
@@ -1,15 +0,0 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class PostToAdd {
private String threadId;
private String postId;
}
@@ -1,14 +0,0 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class ThreadId {
private String threadId;
}
@@ -1,14 +0,0 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class ThreadUrl {
private String url;
}
@@ -8,22 +8,20 @@ import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.domain.DownloadSpeed;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
@Slf4j
public class WebSocketBroadcast {
public class DataBroadcast {
private final SimpMessagingTemplate template;
private final VipergirlsAuthService vipergirlsAuthService;
private final VGAuthService VGAuthService;
private final GlobalStateService globalStateService;
private final DownloadSpeedService downloadSpeedService;
private final DataService dataService;
@@ -31,9 +29,9 @@ public class WebSocketBroadcast {
private final List<Disposable> disposables = new ArrayList<>();
@Autowired
public WebSocketBroadcast(SimpMessagingTemplate template, VipergirlsAuthService vipergirlsAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService) {
public DataBroadcast(SimpMessagingTemplate template, VGAuthService VGAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService) {
this.template = template;
this.vipergirlsAuthService = vipergirlsAuthService;
this.VGAuthService = VGAuthService;
this.globalStateService = globalStateService;
this.downloadSpeedService = downloadSpeedService;
this.dataService = dataService;
@@ -41,10 +39,10 @@ public class WebSocketBroadcast {
@PostConstruct
private void run() {
disposables.add(vipergirlsAuthService.getLoggedInUser()
disposables.add(VGAuthService.getLoggedInUser()
.subscribeOn(Schedulers.io())
.onBackpressureBuffer()
.map(AppDataController.LoggedUser::new)
.map(DataController.LoggedUser::new)
.subscribe(user -> template.convertAndSend("/topic/user", user), e -> log.error("Failed to send data to client", e)));
disposables.add(globalStateService.getLiveGlobalState()
@@ -9,6 +9,8 @@ import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.domain.DownloadSpeed;
import tn.mnlr.vripper.services.domain.GlobalState;
import java.util.Collection;
import java.util.List;
@@ -16,16 +18,16 @@ import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@Controller
public class AppDataController {
public class DataController {
private final VipergirlsAuthService vipergirlsAuthService;
private final VGAuthService VGAuthService;
private final GlobalStateService globalStateService;
private final DownloadSpeedService downloadSpeedService;
private final DataService dataService;
@Autowired
public AppDataController(VipergirlsAuthService vipergirlsAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService) {
this.vipergirlsAuthService = vipergirlsAuthService;
public DataController(VGAuthService VGAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService) {
this.VGAuthService = VGAuthService;
this.globalStateService = globalStateService;
this.downloadSpeedService = downloadSpeedService;
this.dataService = dataService;
@@ -43,7 +45,7 @@ public class AppDataController {
@SubscribeMapping("/user")
public LoggedUser user() {
return new LoggedUser(vipergirlsAuthService.getLoggedUser());
return new LoggedUser(VGAuthService.getLoggedUser());
}
@SubscribeMapping("/download-state")
@@ -1 +0,0 @@
base.dir=${user.home}
@@ -1 +0,0 @@
base.dir=${user.dir}
@@ -2,15 +2,15 @@ logging.level.org.springframework.web=INFO
logging.level.root=INFO
logging.level.org.apache.http=INFO
logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats=ERROR
logging.file.name=${base.dir}/.vripper/vripper.log
logging.file.name=${base.dir}/${base.dir.name}/vripper.log
server.port=${vripper.server.port:8080}
management.endpoints.web.exposure.include=shutdown
management.endpoint.shutdown.enabled=true
server.error.include-message=always
spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.xml
spring.datasource.url=jdbc:hsqldb:file:${base.dir}/.vripper/db/xparty
spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/xparty;hsqldb.lock_file=false
spring.datasource.username=SA
spring.datasource.password=lEtmEIn
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
spring.profiles.active=portable
#spring.profiles.active=installer
base.dir=${user.dir}
base.dir.name=${vripper.base.dir.name:vripper}
@@ -164,4 +164,9 @@
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
referencedColumnNames="ID" referencedTableName="POST"/>
</changeSet>
<changeSet id="1600798657000-01" author="sysgen">
<createIndex tableName="POST" indexName="POST_UQ_POST_ID_IDX">
<column name="POST_ID"/>
</createIndex>
</changeSet>
</databaseChangeLog>
+4 -3
View File
@@ -25,7 +25,8 @@
"tsConfig": "src/tsconfig.app.json",
"assets": [
"src/favicon.ico",
"src/assets"
"src/assets",
{ "glob": "mdi.svg", "input": "./node_modules/@mdi/angular-material", "output": "./assets" }
],
"styles": [
"src/styles.scss"
@@ -42,7 +43,6 @@
"optimization": true,
"outputHashing": "all",
"sourceMap": false,
"extractCss": true,
"namedChunks": false,
"aot": true,
"extractLicenses": true,
@@ -91,7 +91,8 @@
],
"assets": [
"src/favicon.ico",
"src/assets"
"src/assets",
{ "glob": "mdi.svg", "input": "./node_modules/@mdi/angular-material", "output": "./assets" }
]
}
},
+1 -1
View File
@@ -1,5 +1,5 @@
{
"extends": "../tsconfig.base.json",
"extends": "../tsconfig.json",
"compilerOptions": {
"outDir": "../out-tsc/app",
"module": "commonjs",
+2211 -1952
View File
File diff suppressed because it is too large Load Diff
+28 -27
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-ui",
"version": "3.0.0",
"version": "3.2.0",
"scripts": {
"ng": "ng",
"start": "ng serve",
@@ -13,46 +13,47 @@
},
"private": true,
"dependencies": {
"@angular/animations": "10.0.2",
"@angular/cdk": "10.0.1",
"@angular/common": "10.0.2",
"@angular/compiler": "10.0.2",
"@angular/core": "10.0.2",
"@angular/flex-layout": "10.0.0-beta.32",
"@angular/forms": "10.0.2",
"@angular/material": "10.0.1",
"@angular/platform-browser": "10.0.2",
"@angular/platform-browser-dynamic": "10.0.2",
"@angular/router": "10.0.2",
"@angular/animations": "11.0.0",
"@angular/cdk": "11.0.0",
"@angular/common": "11.0.0",
"@angular/compiler": "11.0.0",
"@angular/core": "11.0.0",
"@angular/flex-layout": "11.0.0-beta.33",
"@angular/forms": "11.0.0",
"@angular/material": "11.0.0",
"@angular/platform-browser": "11.0.0",
"@angular/platform-browser-dynamic": "11.0.0",
"@angular/router": "11.0.0",
"@mdi/angular-material": "^5.8.55",
"@stomp/rx-stomp": "^0.3.5",
"ag-grid-angular": "23.2.1",
"ag-grid-angular": "24.1.0",
"ag-grid-community": "23.2.1",
"core-js": "3.6.5",
"ngx-electron": "2.2.0",
"rxjs": "6.5.5",
"tslib": "^2.0.0",
"rxjs": "6.6.3",
"tslib": "^2.0.1",
"zone.js": "0.10.3"
},
"devDependencies": {
"@angular-devkit/build-angular": "^0.1000.0",
"@angular/cli": "10.0.0",
"@angular/compiler-cli": "10.0.2",
"@angular/language-service": "10.0.2",
"@types/jasmine": "3.5.11",
"@angular-devkit/build-angular": "^0.1100.0",
"@angular/cli": "11.0.0",
"@angular/compiler-cli": "11.0.0",
"@angular/language-service": "11.0.0",
"@types/jasmine": "3.5.12",
"@types/jasminewd2": "2.0.8",
"@types/node": "12.12.21",
"codelyzer": "5.2.2",
"electron": "9.1.0",
"jasmine-core": "~3.5.0",
"codelyzer": "6.0.0",
"electron": "10.1.5",
"jasmine-core": "~3.6.0",
"jasmine-spec-reporter": "~5.0.0",
"karma": "~5.0.0",
"karma": "~5.1.1",
"karma-chrome-launcher": "~3.1.0",
"karma-coverage-istanbul-reporter": "~3.0.2",
"karma-jasmine": "~3.3.0",
"karma-jasmine": "~4.0.1",
"karma-jasmine-html-reporter": "^1.5.0",
"protractor": "~7.0.0",
"ts-node": "8.10.2",
"tslint": "~6.1.0",
"typescript": "3.9.5"
"tslint": "~6.1.3",
"typescript": "4.0.5"
}
}
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.0</version>
<version>3.2.0</version>
</parent>
<artifactId>vripper-ui</artifactId>
<name>vripper-ui</name>
+1 -1
View File
@@ -10,7 +10,7 @@ const routes: Routes = [
@NgModule({
imports: [
RouterModule.forRoot(routes)
RouterModule.forRoot(routes, { relativeLinkResolution: 'legacy' })
],
exports: [
RouterModule
+1
View File
@@ -20,3 +20,4 @@
</div>
</div>
</ng-template>
+9 -9
View File
@@ -32,8 +32,8 @@
height: 100%;
position: absolute;
left: 0;
top: 0;
animation: sk-chase-dot 2.0s infinite ease-in-out both;
top: 0;
animation: sk-chase-dot 2.0s infinite ease-in-out both;
}
.sk-chase-dot:before {
@@ -43,7 +43,7 @@
height: 25%;
background-color: #fff;
border-radius: 100%;
animation: sk-chase-dot-before 2.0s infinite ease-in-out both;
animation: sk-chase-dot-before 2.0s infinite ease-in-out both;
}
.sk-chase-dot:nth-child(1) { animation-delay: -1.1s; }
@@ -60,17 +60,17 @@
.sk-chase-dot:nth-child(6):before { animation-delay: -0.6s; }
@keyframes sk-chase {
100% { transform: rotate(360deg); }
100% { transform: rotate(360deg); }
}
@keyframes sk-chase-dot {
80%, 100% { transform: rotate(360deg); }
80%, 100% { transform: rotate(360deg); }
}
@keyframes sk-chase-dot-before {
50% {
transform: scale(0.4);
transform: scale(0.4);
} 100%, 0% {
transform: scale(1.0);
}
}
transform: scale(1.0);
}
}
@@ -8,4 +8,4 @@
background-color: mat-color($background, background);
}
}
}
+3 -3
View File
@@ -1,10 +1,10 @@
import {AppService} from './app.service';
import {ClipboardService} from './clipboard.service';
import {AppService} from './services/app.service';
import {ClipboardService} from './services/clipboard.service';
import {ElectronService} from 'ngx-electron';
import {AfterViewInit, ChangeDetectionStrategy, Component, NgZone, OnDestroy, Renderer2} from '@angular/core';
import {MatDialog} from '@angular/material/dialog';
import {BehaviorSubject, merge, Subject, Subscription} from 'rxjs';
import {WsConnectionService} from './ws-connection.service';
import {WsConnectionService} from './services/ws-connection.service';
import {RxStompState} from '@stomp/rx-stomp';
@Component({
+17 -29
View File
@@ -1,62 +1,50 @@
import {PostContextMenuComponent} from './posts/context-menu/post-context-menu.component';
import {PostDetailsContextMenuComponent} from './post-progress/context-menu/post-details-context-menu.component';
import {AppPreviewDirective} from './domain/preview-tooltip.directive';
import {AppPreviewDirective} from './preview-tooltip/preview-tooltip.directive';
import {BrowserAnimationsModule} from '@angular/platform-browser/animations';
import {NgModule} from '@angular/core';
import {AppComponent} from './app.component';
import {MaterialModule} from './material.module';
import {PostsComponent} from './posts/posts.component';
import {PostDetailComponent} from './post-progress/post-detail.component';
import {PhotosComponent} from './photos/photos.component';
import {FlexLayoutModule} from '@angular/flex-layout';
import {AppRoutingModule} from './app-routing.module';
import {HttpClientModule} from '@angular/common/http';
import {FormsModule, ReactiveFormsModule} from '@angular/forms';
import {AgGridModule} from 'ag-grid-angular';
import {PostProgressRendererComponent} from './posts/renderer/post-progress.renderer.component';
import {PostDetailsProgressRendererComponent} from './post-progress/renderer/post-details-progress.component';
import {LoginComponent} from './login/login.component';
import {HomeComponent} from './home/home.component';
import {SettingsComponent} from './settings/settings.component';
import {ConfirmDialogComponent} from './confirmation-component/confirmation-dialog';
import {MultiPostComponent} from './multi-post/multi-post.component';
import {MultiPostItemsComponent} from './multi-post-items/multi-post-items.component';
import {OverlayModule} from '@angular/cdk/overlay';
import {AppPreviewComponent} from './domain/preview-tooltip.component';
import {UrlRendererComponent} from './multi-post/renderer/url-renderer.component';
import {FilterComponent} from './filter/filter.component';
import {AppPreviewComponent} from './preview-tooltip/preview-tooltip.component';
import {ScanComponent} from './scan/scan.component';
import {StatusBarComponent} from './status-bar/status-bar.component';
import {ToolbarComponent} from './toolbar/toolbar.component';
import {GrabQueueComponent} from './grab-queue/grab-queue.component';
import {UrlGrabRendererComponent} from './grab-queue/renderer/url-renderer.component';
import {MultiPostGridComponent} from './multi-post-grid/multi-post-grid.component';
import {AlternativeTitleComponent} from './posts/alternative-title/alternative-title.component';
import {NgxElectronModule} from 'ngx-electron';
import {MatIconRegistry} from '@angular/material/icon';
import {DomSanitizer} from '@angular/platform-browser';
@NgModule({
declarations: [
AppComponent,
PostsComponent,
PostDetailComponent,
PostProgressRendererComponent,
PostDetailsProgressRendererComponent,
LoginComponent,
PhotosComponent,
HomeComponent,
SettingsComponent,
ConfirmDialogComponent,
MultiPostComponent,
MultiPostItemsComponent,
AppPreviewComponent,
AppPreviewDirective,
UrlRendererComponent,
UrlGrabRendererComponent,
FilterComponent,
ScanComponent,
StatusBarComponent,
ToolbarComponent,
GrabQueueComponent,
MultiPostGridComponent,
PostContextMenuComponent,
PostDetailsContextMenuComponent,
AlternativeTitleComponent
AlternativeTitleComponent,
],
imports: [
BrowserAnimationsModule,
@@ -66,16 +54,16 @@ import {NgxElectronModule} from 'ngx-electron';
FlexLayoutModule,
AppRoutingModule,
ReactiveFormsModule,
AgGridModule.withComponents([
PostProgressRendererComponent,
PostDetailsProgressRendererComponent,
UrlRendererComponent,
UrlGrabRendererComponent
]),
AgGridModule,
OverlayModule,
NgxElectronModule
],
bootstrap: [AppComponent]
})
export class AppModule {
constructor(matIconRegistry: MatIconRegistry, domSanitizer: DomSanitizer) {
matIconRegistry.addSvgIconSet(
domSanitizer.bypassSecurityTrustResourceUrl('./assets/mdi.svg')
);
}
}
@@ -1,3 +1,4 @@
export class GlobalState {
constructor(public running: number, public queued: number, public remaining: number, public error: number) {}
constructor(public running: number, public remaining: number, public error: number) {
}
}
@@ -1,4 +1,4 @@
export class GrabQueueState {
export class MultiPostModel {
constructor(
public type: string,
public link: string,
@@ -1,4 +1,4 @@
export class PostDetails {
export class Photo {
constructor(
public postId: string,
public url: string,
@@ -1,4 +1,4 @@
export class PostState {
export class Post {
constructor(
public postId: string,
public title: string,
@@ -9,6 +9,5 @@ export interface Settings {
vThanks: boolean;
desktopClipboard: boolean;
viewPhotos: boolean;
notification: boolean;
resolveTitle: boolean;
}
@@ -1,4 +1,4 @@
export class VRPostParse {
export class MultiPostItem {
constructor(
public threadId: string,
public postId: string,
@@ -1,3 +0,0 @@
<mat-chip-list>
<mat-chip (click)="select(c)" *ngFor="let c of criteria" [selected]="c.selected" color="accent">{{c.value}}</mat-chip>
</mat-chip-list>
@@ -1,26 +0,0 @@
import { Component, OnInit } from '@angular/core';
@Component({
selector: 'app-filter',
templateUrl: './filter.component.html',
styleUrls: ['./filter.component.scss']
})
export class FilterComponent implements OnInit {
constructor() { }
criteria = [
{value: 'All', selected: true},
{value: 'Downloading', selected: false},
{value: 'Completed', selected: false},
{value: 'Error', selected: false},
];
select(chip: {value: string, selected: boolean}) {
this.criteria.forEach(e => e.selected = false);
chip.selected = true;
}
ngOnInit() {
}
}
@@ -1,4 +0,0 @@
<div style="width: 100%; height: 100%; padding-top: 5px">
<ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-alpine" style="width: 100%; height: calc(100% - 5px);">
</ag-grid-angular>
</div>

Some files were not shown because too many files have changed in this diff Show More