Compare commits

..
28 Commits
Author SHA1 Message Date
death-claw d5ae5d57d1 v3.0.4 2020-08-16 16:11:11 +01:00
death-claw 537d76d7be Improve rename UI 2020-08-16 16:06:17 +01:00
death-claw c1247faacc v3.0.3 2020-08-16 11:29:01 +01:00
death-claw efcc685f30 Remove notification feature 2020-08-16 11:26:57 +01:00
death-claw c825b22ce4 Fix some bugs with MacOS 2020-08-16 10:58:19 +01:00
death-claw 6ef5d5474f Fix some bugs with MacOS 2020-08-16 10:20:56 +01:00
death-claw bd25801c6c Fix base dir configuration for MacOS 2020-08-16 00:18:18 +01:00
death-claw 377b93c1ac Fix base dir configuration for AppImage 2020-08-16 00:02:18 +01:00
death-claw d498d20f6b Set default download path to user's home 2020-08-15 23:23:43 +01:00
death-claw d18544d414 Fix bin location for mac 2020-08-15 23:10:58 +01:00
death-claw 3c872b79bd Update app data configuration 2020-08-15 23:00:48 +01:00
death-claw 37900cac49 v3.0.2 2020-08-15 15:38:53 +01:00
death-claw be57c76d93 Fix bugs with download queue 2020-08-15 15:36:47 +01:00
death-claw 8b52ad7858 v3.0.1 2020-08-03 18:48:45 +01:00
death-claw a0976bde02 Fix bugs with download queue 2020-08-03 18:43:42 +01:00
death-claw 15c97d5cfc v3.0.0 2020-08-03 15:06:36 +01:00
death-claw 905cbd5a35 Code refactoring
Fix bugs with rename gallery
2020-08-03 15:03:42 +01:00
death-clawandGitHub 5c0e2e6185 Merge pull request #12 from life-claw/imgVenueFix
Fixed ImageVenue
2020-07-30 20:56:41 +01:00
death-clawandGitHub 4836d2dacb Merge branch 'master' into imgVenueFix 2020-07-30 20:56:30 +01:00
death-claw bc81ec1ab5 Bug fixes
Code enhancements
2020-07-30 14:59:58 +01:00
death-claw 8a4d039a37 Bug fixes
Code enhancements
2020-07-30 14:00:56 +01:00
death-claw 468639264a Major rewrites of data persistence, websocket, and user interface 2020-07-26 15:10:09 +01:00
death-claw 9b649b470f Major rewrites of data persistence, websocket, and user interface 2020-07-26 14:45:04 +01:00
life-claw edd5cf2bf7 Fixed ImageVenue
Updated XPaths to match their current markup
And added HTTPs redirect, since they support that now
2020-07-17 16:57:51 -05:00
death-claw 67c605e640 v2.12.1 2020-05-22 15:55:05 +01:00
death-claw 8d4b878656 Fix alternative name bug 2020-05-22 15:54:11 +01:00
death-claw bf532dd1d0 v2.12.0 2020-05-21 16:43:03 +01:00
death-claw eb1a125e59 Use new user hash from the api 2020-05-21 16:41:16 +01:00
151 changed files with 6709 additions and 7368 deletions
+2 -1
View File
@@ -3,4 +3,5 @@
/**/*/target
/**/*/build-dir
/**/*/java-runtime
.idea
.idea
.vripper
+29
View File
@@ -1,5 +1,34 @@
# Changelog
## [3.0.4] - 2020-08-16
### Changed
- Improve rename UI
## [3.0.3] - 2020-08-16
### Changed
- Fix bugs with electron app
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
- Introducing a proper database for persistence
## [2.12.1] - 2020-05-22
### Changed
- Fix alternative name bug
## [2.12.0] - 2020-05-21
### Changed
- Use new user hash from the api
## [2.11.7] - 2020-05-08
### Changed
- Fix bug with paths creation
+2 -2
View File
@@ -4,12 +4,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>2.11.7</version>
<version>3.0.4</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.1.RELEASE</version>
<version>2.3.0.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<modules>
+65 -51
View File
@@ -1,13 +1,16 @@
require('v8-compile-cache');
const { app, BrowserWindow } = require("electron");
const {app, BrowserWindow} = require("electron");
const path = require("path");
const url = require("url");
const getPort = require("get-port");
const { spawn } = require("child_process");
const { ipcMain } = require("electron");
const { dialog } = require("electron");
const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const appDir = process.env.APPDIR;
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
const appImagePath = process.env.APPIMAGE;
let win;
let vripperServer;
@@ -23,14 +26,14 @@ process.on("uncaughtException", err => {
process.exit(1);
});
function createWindow() {
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId("tn.mnlr.vripper");
}
let icon;
if(process.platform === "win32") {
if (process.platform === "win32") {
icon = __dirname + '/icon.ico';
} else {
icon = __dirname + '/icon.png';
@@ -41,7 +44,8 @@ function createWindow() {
minWidth: 800,
minHeight: 600,
webPreferences: {
nodeIntegration: true
nodeIntegration: true,
enableRemoteModule: true
},
icon: icon
});
@@ -65,6 +69,33 @@ function createWindow() {
});
}
shutdownServer = () => {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: {'content-type': 'application/json'},
}).then((response) => {
terminationInteval = setInterval(() => {
terminationAttemps++;
if (terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if (terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
}).catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
@@ -75,29 +106,30 @@ if (!gotTheLock) {
ipcMain.on("get-port", event => {
event.reply("port", port);
});
let javaBinPath;
if(appDir !== undefined) {
javaBinPath = path.join(appDir, "java-runtime/bin/java");
const appPath = path.join(app.getAppPath(), '../../');
let javaBinPath, jarPath, baseDir;
if (appImageDir !== undefined && process.platform === 'linux') {
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
baseDir = path.join(appImagePath, '..');
} else if (process.platform === 'darwin') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = path.join(appPath, '../..');
} else if (process.platform === 'win32') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = appPath;
} else {
if(process.platform === 'darwin') {
javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
} else {
javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
}
}
let jarPath;
if(appDir !== undefined) {
jarPath = path.join(appDir, "bin/vripper-server.jar");
} else {
if(process.platform === 'darwin') {
jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
} else {
jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
}
console.error(`Unknown platform ${process.platform}`);
app.quit();
}
vripperServer = spawn(javaBinPath, [
"-Xms256m",
"-Dvripper.server.port=" + port,
"-Dbase.dir=" + baseDir,
"-jar",
jarPath
], {
@@ -121,31 +153,13 @@ if (!gotTheLock) {
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: { 'content-type': 'application/json' },
}).then((response) => {
terminationInteval= setInterval(() => {
terminationAttemps++;
if(terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if(terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
})
.catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
shutdownServer();
}
});
app.on("will-quit", () => {
if (process.platform === "darwin") {
shutdownServer();
}
});
+230 -297
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "2.11.7",
"version": "3.0.4",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@@ -21,9 +21,9 @@
}
},
"@electron/get": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/@electron/get/-/get-1.10.0.tgz",
"integrity": "sha512-hlueNXU51c3CwQjBw/i5fwt+VfQgSQVUTdicpCHkhEjNZaa4CXJ5W1GaxSwtLE2dvRmAHjpIjUMHTqJ53uojfg==",
"version": "1.12.2",
"resolved": "https://registry.npmjs.org/@electron/get/-/get-1.12.2.tgz",
"integrity": "sha512-vAuHUbfvBQpYTJ5wB7uVIDq5c/Ry0fiTBMs7lnEYAo/qXXppIVcWdfBr57u6eRnKdVso7KSiH6p/LbQAG6Izrg==",
"dev": true,
"requires": {
"debug": "^4.1.1",
@@ -64,9 +64,9 @@
"dev": true
},
"@types/fs-extra": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.0.tgz",
"integrity": "sha512-UoOfVEzAUpeSPmjm7h1uk5MH6KZma2z2O7a75onTGjnNvAvMVrPzPL/vBbT65iIGHWj6rokwfmYcmxmlSf2uwg==",
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-B42Sxuaz09MhC3DDeW5kubRcQ5by4iuVQ0cRRWM2lggLzAa/KVom0Aft/208NgMvNQQZ86s5rVcqDdn/SH0/mg==",
"dev": true,
"requires": {
"@types/node": "*"
@@ -78,9 +78,9 @@
"integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q=="
},
"@types/yargs": {
"version": "15.0.4",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.4.tgz",
"integrity": "sha512-9T1auFmbPZoxHz0enUFlUuKRy3it01R+hlggyVUMtnCTQRunsQYifnSGb8hET4Xo8yiC0o0r1paW3ud5+rbURg==",
"version": "15.0.5",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.5.tgz",
"integrity": "sha512-Dk/IDOPtOgubt/IaevIUbTgV7doaKkoorvOyYM2CMwuDyP89bekI7H4xLIwunNYiK9jhCkmc6pUrJk3cj2AB9w==",
"dev": true,
"requires": {
"@types/yargs-parser": "*"
@@ -93,9 +93,9 @@
"dev": true
},
"ajv": {
"version": "6.12.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz",
"integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==",
"version": "6.12.3",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz",
"integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==",
"dev": true,
"requires": {
"fast-deep-equal": "^3.1.1",
@@ -105,9 +105,9 @@
}
},
"ajv-keywords": {
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.4.1.tgz",
"integrity": "sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ==",
"version": "3.5.1",
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.1.tgz",
"integrity": "sha512-KWcq3xN8fDjSB+IMoh2VaXVhRI0BBGxoYp3rx7Pkb6z0cFjYR9Q9l4yZqqals0/zsioCmocC5H6UvsGD4MoIBA==",
"dev": true
},
"ansi-align": {
@@ -142,39 +142,38 @@
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
"integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
"dev": true,
"requires": {
"@types/color-name": "^1.1.1",
"color-convert": "^2.0.1"
}
},
"app-builder-bin": {
"version": "3.5.8",
"resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-3.5.8.tgz",
"integrity": "sha512-ni3q7QTfQNWHNWuyn5x3FZu6GnQZv+TFnfgk5++svqleKEhHGqS1mIaKsh7x5pBX6NFXU3/+ktk98wA/AW4EXw==",
"version": "3.5.9",
"resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-3.5.9.tgz",
"integrity": "sha512-NSjtqZ3x2kYiDp3Qezsgukx/AUzKPr3Xgf9by4cYt05ILWGAptepeeu0Uv+7MO+41o6ujhLixTou8979JGg2Kg==",
"dev": true
},
"app-builder-lib": {
"version": "22.6.0",
"resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-22.6.0.tgz",
"integrity": "sha512-ky2aLYy92U+Gh6dKq/e8/bNmCotp6/GMhnX8tDZPv9detLg9WuBnWWi1ktBPlpbl1DREusy+TIh+9rgvfduQoA==",
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-22.7.0.tgz",
"integrity": "sha512-blRKwV8h0ztualXS50ciCTo39tbuDGNS+ldcy8+KLvKXuT6OpYnSJ7M6MSfPT+xWatshMHJV1rJx3Tl+k/Sn/g==",
"dev": true,
"requires": {
"7zip-bin": "~5.0.3",
"@develar/schema-utils": "~2.6.5",
"async-exit-hook": "^2.0.1",
"bluebird-lst": "^1.0.9",
"builder-util": "22.6.0",
"builder-util-runtime": "8.7.0",
"builder-util": "22.7.0",
"builder-util-runtime": "8.7.1",
"chromium-pickle-js": "^0.2.0",
"debug": "^4.1.1",
"ejs": "^3.1.2",
"electron-publish": "22.6.0",
"debug": "^4.2.0",
"ejs": "^3.1.3",
"electron-publish": "22.7.0",
"fs-extra": "^9.0.0",
"hosted-git-info": "^3.0.4",
"is-ci": "^2.0.0",
"isbinaryfile": "^4.0.6",
"js-yaml": "^3.13.1",
"js-yaml": "^3.14.0",
"lazy-val": "^1.0.4",
"minimatch": "^3.0.4",
"normalize-package-data": "^2.5.0",
@@ -184,10 +183,19 @@
"temp-file": "^3.3.7"
},
"dependencies": {
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"dev": true,
"requires": {
"ms": "2.1.2"
}
},
"fs-extra": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.0.tgz",
"integrity": "sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==",
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
"dev": true,
"requires": {
"at-least-node": "^1.0.0",
@@ -255,12 +263,11 @@
"dev": true
},
"axios": {
"version": "0.19.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz",
"integrity": "sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==",
"version": "0.19.2",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz",
"integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==",
"requires": {
"follow-redirects": "1.5.10",
"is-buffer": "^2.0.2"
"follow-redirects": "1.5.10"
}
},
"balanced-match": {
@@ -351,31 +358,40 @@
"dev": true
},
"builder-util": {
"version": "22.6.0",
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.6.0.tgz",
"integrity": "sha512-jgdES2ExJYkuXC3DEaGAjFctKNA81C4QDy8zdoc+rqdSqheTizuDNtZg02uMFklmUES4V4fggmqds+Y7wraqng==",
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.7.0.tgz",
"integrity": "sha512-UV3MKL0mwjMq2y9JlBf28Cegpj0CrIXcjGkO0TXn+QZ6Yy9rY6lHOuUvpQ19ct2Qh1o+QSwH3Q1nKUf5viJBBg==",
"dev": true,
"requires": {
"7zip-bin": "~5.0.3",
"@types/debug": "^4.1.5",
"@types/fs-extra": "^8.1.0",
"app-builder-bin": "3.5.8",
"@types/fs-extra": "^9.0.1",
"app-builder-bin": "3.5.9",
"bluebird-lst": "^1.0.9",
"builder-util-runtime": "8.7.0",
"builder-util-runtime": "8.7.1",
"chalk": "^4.0.0",
"debug": "^4.1.1",
"debug": "^4.2.0",
"fs-extra": "^9.0.0",
"is-ci": "^2.0.0",
"js-yaml": "^3.13.1",
"js-yaml": "^3.14.0",
"source-map-support": "^0.5.19",
"stat-mode": "^1.0.0",
"temp-file": "^3.3.7"
},
"dependencies": {
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"dev": true,
"requires": {
"ms": "2.1.2"
}
},
"fs-extra": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.0.tgz",
"integrity": "sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==",
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
"dev": true,
"requires": {
"at-least-node": "^1.0.0",
@@ -403,13 +419,24 @@
}
},
"builder-util-runtime": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-8.7.0.tgz",
"integrity": "sha512-G1AqqVM2vYTrSFR982c1NNzwXKrGLQjVjaZaWQdn4O6Z3YKjdMDofw88aD9jpyK9ZXkrCxR0tI3Qe9wNbyTlXg==",
"version": "8.7.1",
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-8.7.1.tgz",
"integrity": "sha512-uEBH1nAnTvzjcsrh2XI3qOzJ39h0+9kuIuwj+kCc3a07TZNGShfJcai8fFzL3mNgGjEFxoq+XMssR11r+FOFSg==",
"dev": true,
"requires": {
"debug": "^4.1.1",
"debug": "^4.2.0",
"sax": "^1.2.4"
},
"dependencies": {
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"dev": true,
"requires": {
"ms": "2.1.2"
}
}
}
},
"cacheable-request": {
@@ -451,9 +478,9 @@
"dev": true
},
"chalk": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.0.0.tgz",
"integrity": "sha512-N9oWFcegS0sFr9oh1oz2d7Npos6vNoWW9HvtCg5N1KRFpUhaAhvTv5Y58g880fZaEYSNm3qDz8SU1UrGvp+n7A==",
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
"integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
"dev": true,
"requires": {
"ansi-styles": "^4.1.0",
@@ -498,41 +525,6 @@
"requires": {
"slice-ansi": "^3.0.0",
"string-width": "^4.2.0"
},
"dependencies": {
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"string-width": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
"integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.0"
}
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"requires": {
"ansi-regex": "^5.0.0"
}
}
}
},
"cliui": {
@@ -576,7 +568,6 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"requires": {
"color-name": "~1.1.4"
}
@@ -584,8 +575,7 @@
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"concat-map": {
"version": "0.0.1",
@@ -630,9 +620,9 @@
}
},
"copy-dir": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/copy-dir/-/copy-dir-1.1.0.tgz",
"integrity": "sha512-eqHj1AMmS53QiLr8Xqcz754m6ljmG9eFnrzMhPGA1eKrsNYBGIKLktXz6JnzqNMSH/6Qksxgq8OMKsr0MPjKlw=="
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/copy-dir/-/copy-dir-1.3.0.tgz",
"integrity": "sha512-Q4+qBFnN4bwGwvtXXzbp4P/4iNk0MaiGAzvQ8OiMtlLjkIKjmNN689uVzShSM0908q7GoFHXIPx4zi75ocoaHw=="
},
"core-js": {
"version": "3.6.5",
@@ -723,23 +713,23 @@
"optional": true
},
"dmg-builder": {
"version": "22.6.0",
"resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-22.6.0.tgz",
"integrity": "sha512-rJxuGhHIpcuDGBtWZMM8aLxkbZNgYO2MO5dUerDIBXebhX1K8DA23iz/uZ8ahcRNgWEv57b8GDqJbXKEfr5T0A==",
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-22.7.0.tgz",
"integrity": "sha512-5Ea2YEz6zSNbyGzZD+O9/MzmaXb6oa15cSKWo4JQ1xP4rorOpte7IOj2jcwYjtc+Los2gu1lvT314OC1OZIWgg==",
"dev": true,
"requires": {
"app-builder-lib": "22.6.0",
"builder-util": "22.6.0",
"app-builder-lib": "22.7.0",
"builder-util": "22.7.0",
"fs-extra": "^9.0.0",
"iconv-lite": "^0.5.1",
"js-yaml": "^3.13.1",
"js-yaml": "^3.14.0",
"sanitize-filename": "^1.6.3"
},
"dependencies": {
"fs-extra": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.0.tgz",
"integrity": "sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==",
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
"dev": true,
"requires": {
"at-least-node": "^1.0.0",
@@ -825,18 +815,18 @@
"dev": true
},
"ejs": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.2.tgz",
"integrity": "sha512-zFuywxrAWtX5Mk2KAuoJNkXXbfezpNA0v7i+YC971QORguPekpjpAgeOv99YWSdKXwj7JxI2QAWDeDkE8fWtXw==",
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.3.tgz",
"integrity": "sha512-wmtrUGyfSC23GC/B1SMv2ogAUgbQEtDmTIhfqielrG5ExIM9TP4UoYdi90jLF1aTcsWCJNEO0UrgKzP0y3nTSg==",
"dev": true,
"requires": {
"jake": "^10.6.1"
}
},
"electron": {
"version": "8.2.4",
"resolved": "https://registry.npmjs.org/electron/-/electron-8.2.4.tgz",
"integrity": "sha512-Lle0InIgSAHZxD5KDY0wZ1A2Zlc6GHwMhAxoHMzn05mndyP1YBkCYHc0TDDofzUTrsLFofduPjlknO5Oj9fTPA==",
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/electron/-/electron-9.1.0.tgz",
"integrity": "sha512-VRAF8KX1m0py9I9sf0kw1kWfeC87mlscfFcbcRdLBsNJ44/GrJhi3+E8rKbpHUeZNQxsPaVA5Zu5Lxb6dV/scQ==",
"dev": true,
"requires": {
"@electron/get": "^1.0.1",
@@ -845,26 +835,26 @@
},
"dependencies": {
"@types/node": {
"version": "12.12.37",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.37.tgz",
"integrity": "sha512-4mXKoDptrXAwZErQHrLzpe0FN/0Wmf5JRniSVIdwUrtDf9wnmEV1teCNLBo/TwuXhkK/bVegoEn/wmb+x0AuPg==",
"version": "12.12.50",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.50.tgz",
"integrity": "sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w==",
"dev": true
}
}
},
"electron-builder": {
"version": "22.6.0",
"resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-22.6.0.tgz",
"integrity": "sha512-aLHlB6DTfjJ3MI4AUIFeWnwIozNgNlbOk2c2sTHxB10cAKp0dBVSPZ7xF5NK0uwDhElvRzJQubnHtJD6zKg42Q==",
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-22.7.0.tgz",
"integrity": "sha512-t6E3oMutpST64YWbZCg7HodEwJOsnjUF1vnDIHm2MW6CFZPX8tlCK6efqaV66LU0E0Nkp/JH6TE5bCqQ1+VdPQ==",
"dev": true,
"requires": {
"@types/yargs": "^15.0.4",
"app-builder-lib": "22.6.0",
"@types/yargs": "^15.0.5",
"app-builder-lib": "22.7.0",
"bluebird-lst": "^1.0.9",
"builder-util": "22.6.0",
"builder-util-runtime": "8.7.0",
"builder-util": "22.7.0",
"builder-util-runtime": "8.7.1",
"chalk": "^4.0.0",
"dmg-builder": "22.6.0",
"dmg-builder": "22.7.0",
"fs-extra": "^9.0.0",
"is-ci": "^2.0.0",
"lazy-val": "^1.0.4",
@@ -875,9 +865,9 @@
},
"dependencies": {
"fs-extra": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.0.tgz",
"integrity": "sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==",
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
"dev": true,
"requires": {
"at-least-node": "^1.0.0",
@@ -905,50 +895,50 @@
}
},
"electron-context-menu": {
"version": "0.13.0",
"resolved": "https://registry.npmjs.org/electron-context-menu/-/electron-context-menu-0.13.0.tgz",
"integrity": "sha512-a98UDykOn+tiyb2mQEz710ZNWj/L85wHv6jRUJFE9GNLSaRH5I5BR022RYoWInLTj1Mns66vh9SueyMPWc+aTQ==",
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/electron-context-menu/-/electron-context-menu-2.1.0.tgz",
"integrity": "sha512-xnJS4C24W/h6rw2fbRYMiHCHWzbJeuhPTLJR88lJ+jSnO8VKY0oOolE2AeL/zTTiUYrOXF+3POcfpSoaF0d8Pw==",
"requires": {
"cli-truncate": "^2.0.0",
"electron-dl": "^1.2.0",
"electron-dl": "^3.0.0",
"electron-is-dev": "^1.0.1"
}
},
"electron-dl": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/electron-dl/-/electron-dl-1.14.0.tgz",
"integrity": "sha512-4okyei42a1mLsvLK7hLrIfd20EQzB18nIlLTwBV992aMSmTGLUEFRTmO1MfSslGNrzD8nuPuy1l/VxO8so4lig==",
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/electron-dl/-/electron-dl-3.0.1.tgz",
"integrity": "sha512-JHEsUCusT7x/i682LNl1gJYWTiic71Sp3ykdvwbgywGuY19dLUd3VqkE5zI905e4Wy+G/pDcLjoeFo0SgCm3CA==",
"requires": {
"ext-name": "^5.0.0",
"pupa": "^1.0.0",
"unused-filename": "^1.0.0"
"pupa": "^2.0.1",
"unused-filename": "^2.1.0"
}
},
"electron-is-dev": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/electron-is-dev/-/electron-is-dev-1.1.0.tgz",
"integrity": "sha512-Z1qA/1oHNowGtSBIcWk0pcLEqYT/j+13xUw/MYOrBUOL4X7VN0i0KCTf5SqyvMPmW5pSPKbo28wkxMxzZ20YnQ=="
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/electron-is-dev/-/electron-is-dev-1.2.0.tgz",
"integrity": "sha512-R1oD5gMBPS7PVU8gJwH6CtT0e6VSoD0+SzSnYpNm+dBkcijgA+K7VAMHDfnRq/lkKPZArpzplTW6jfiMYosdzw=="
},
"electron-publish": {
"version": "22.6.0",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-22.6.0.tgz",
"integrity": "sha512-+v05SBf9qR7Os5au+fifloNHy5QxHQkUGudBj68YaTb43Pn37UkwRxSc49Lf13s4wW32ohM45g8BOVInPJEdnA==",
"version": "22.7.0",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-22.7.0.tgz",
"integrity": "sha512-hmU69xlb6vvAV3QfpHYDlkdZMFdBAgDbptoxbLFrnTq5bOkcL8AaDbvxeoZ4+lvqgs29NwqGpkHo2oN+p/hCfg==",
"dev": true,
"requires": {
"@types/fs-extra": "^8.1.0",
"@types/fs-extra": "^9.0.1",
"bluebird-lst": "^1.0.9",
"builder-util": "22.6.0",
"builder-util-runtime": "8.7.0",
"builder-util": "22.7.0",
"builder-util-runtime": "8.7.1",
"chalk": "^4.0.0",
"fs-extra": "^9.0.0",
"lazy-val": "^1.0.4",
"mime": "^2.4.4"
"mime": "^2.4.5"
},
"dependencies": {
"fs-extra": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.0.tgz",
"integrity": "sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==",
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
"integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
"dev": true,
"requires": {
"at-least-node": "^1.0.0",
@@ -1018,13 +1008,12 @@
"escape-goat": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz",
"integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==",
"dev": true
"integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q=="
},
"escape-string-regexp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz",
"integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==",
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
"integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
"dev": true,
"optional": true
},
@@ -1081,9 +1070,9 @@
}
},
"fast-deep-equal": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
"integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==",
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
"dev": true
},
"fast-json-stable-stringify": {
@@ -1118,14 +1107,6 @@
"requires": {
"locate-path": "^5.0.0",
"path-exists": "^4.0.0"
},
"dependencies": {
"path-exists": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
"dev": true
}
}
},
"follow-redirects": {
@@ -1174,12 +1155,9 @@
"dev": true
},
"get-port": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.0.0.tgz",
"integrity": "sha512-imzMU0FjsZqNa6BqOjbbW6w5BivHIuQKopjpPqcnx0AVHJQKCxK1O+Ab3OrVXhrekqfVMjwA9ZYu062R+KcIsQ==",
"requires": {
"type-fest": "^0.3.0"
}
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
"integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ=="
},
"get-stream": {
"version": "4.1.0",
@@ -1191,9 +1169,9 @@
}
},
"glob": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
"integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@@ -1204,19 +1182,19 @@
}
},
"global-agent": {
"version": "2.1.8",
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-2.1.8.tgz",
"integrity": "sha512-VpBe/rhY6Rw2VDOTszAMNambg+4Qv8j0yiTNDYEXXXxkUNGWLHp8A3ztK4YDBbFNcWF4rgsec6/5gPyryya/+A==",
"version": "2.1.12",
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-2.1.12.tgz",
"integrity": "sha512-caAljRMS/qcDo69X9BfkgrihGUgGx44Fb4QQToNQjsiWh+YlQ66uqYVAdA8Olqit+5Ng0nkz09je3ZzANMZcjg==",
"dev": true,
"optional": true,
"requires": {
"boolean": "^3.0.0",
"core-js": "^3.6.4",
"boolean": "^3.0.1",
"core-js": "^3.6.5",
"es6-error": "^4.1.1",
"matcher": "^2.1.0",
"roarr": "^2.15.2",
"semver": "^7.1.2",
"serialize-error": "^5.0.0"
"matcher": "^3.0.0",
"roarr": "^2.15.3",
"semver": "^7.3.2",
"serialize-error": "^7.0.1"
}
},
"global-dirs": {
@@ -1289,12 +1267,12 @@
"dev": true
},
"hosted-git-info": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.4.tgz",
"integrity": "sha512-4oT62d2jwSDBbLLFLZE+1vPuQ1h8p9wjrJ8Mqx5TjsyWmBMV5B13eJqn8pvluqubLf3cJPTfiYCIwNwDNmzScQ==",
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.5.tgz",
"integrity": "sha512-i4dpK6xj9BIpVOTboXIlKG9+8HMKggcrMX7WA24xZtKwX0TPelq/rbaS5rCKeNX8sJXZJGdSxpnEGtta+wismQ==",
"dev": true,
"requires": {
"lru-cache": "^5.1.1"
"lru-cache": "^6.0.0"
}
},
"htmlparser2": {
@@ -1337,9 +1315,9 @@
"dev": true
},
"iconv-lite": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.1.tgz",
"integrity": "sha512-ONHr16SQvKZNSqjQT9gy5z24Jw+uqfO02/ngBSBoqChZ+W8qXX7GPRa1RoUnzGADw8K63R1BXUMzarCVQBpY8Q==",
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz",
"integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==",
"dev": true,
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
@@ -1377,11 +1355,6 @@
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
"dev": true
},
"is-buffer": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz",
"integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw=="
},
"is-ci": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
@@ -1455,9 +1428,9 @@
"dev": true
},
"jake": {
"version": "10.6.1",
"resolved": "https://registry.npmjs.org/jake/-/jake-10.6.1.tgz",
"integrity": "sha512-pHUK3+V0BjOb1XSi95rbBksrMdIqLVC9bJqDnshVyleYsET3H0XAq+3VB2E3notcYvv4wRdRHn13p7vobG+wfQ==",
"version": "10.8.2",
"resolved": "https://registry.npmjs.org/jake/-/jake-10.8.2.tgz",
"integrity": "sha512-eLpKyrfG3mzvGE2Du8VoPbeSkRry093+tyNjdYaBbJS9v17knImYGNXQCUV0gLxQtF82m3E8iRb/wdSQZLoq7A==",
"dev": true,
"requires": {
"async": "0.9.x",
@@ -1525,9 +1498,9 @@
}
},
"js-yaml": {
"version": "3.13.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
"integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz",
"integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==",
"dev": true,
"requires": {
"argparse": "^1.0.7",
@@ -1616,12 +1589,12 @@
"dev": true
},
"lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"requires": {
"yallist": "^3.0.2"
"yallist": "^4.0.0"
}
},
"make-dir": {
@@ -1642,25 +1615,25 @@
}
},
"matcher": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/matcher/-/matcher-2.1.0.tgz",
"integrity": "sha512-o+nZr+vtJtgPNklyeUKkkH42OsK8WAfdgaJE2FNxcjLPg+5QbeEoT6vRj8Xq/iv18JlQ9cmKsEu0b94ixWf1YQ==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
"dev": true,
"optional": true,
"requires": {
"escape-string-regexp": "^2.0.0"
"escape-string-regexp": "^4.0.0"
}
},
"mime": {
"version": "2.4.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
"integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==",
"version": "2.4.6",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz",
"integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==",
"dev": true
},
"mime-db": {
"version": "1.40.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
"integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA=="
"version": "1.44.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
"integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg=="
},
"mimic-response": {
"version": "1.0.1",
@@ -1827,9 +1800,9 @@
}
},
"path-exists": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
"integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU="
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
},
"path-is-absolute": {
"version": "1.0.1",
@@ -1897,9 +1870,12 @@
"dev": true
},
"pupa": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/pupa/-/pupa-1.0.0.tgz",
"integrity": "sha1-mpVopa9+ZXuEYqbp1TKHQ1YM7/Y="
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/pupa/-/pupa-2.0.1.tgz",
"integrity": "sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA==",
"requires": {
"escape-goat": "^2.0.0"
}
},
"rc": {
"version": "1.2.8",
@@ -1990,9 +1966,9 @@
}
},
"rimraf": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
"integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"requires": {
"glob": "^7.1.3"
}
@@ -2069,22 +2045,13 @@
}
},
"serialize-error": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-5.0.0.tgz",
"integrity": "sha512-/VtpuyzYf82mHYTtI4QKtwHa79vAdU5OQpNPAmE/0UDdlGT0ZxHwC+J6gXkw29wwoVI8fMPsfcVHOwXtUQYYQA==",
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
"integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
"dev": true,
"optional": true,
"requires": {
"type-fest": "^0.8.0"
},
"dependencies": {
"type-fest": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
"integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==",
"dev": true,
"optional": true
}
"type-fest": "^0.13.1"
}
},
"set-blocking": {
@@ -2109,28 +2076,6 @@
"is-fullwidth-code-point": "^3.0.0"
},
"dependencies": {
"ansi-styles": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.0.tgz",
"integrity": "sha512-7kFQgnEaMdRtwf6uSfUnVr9gSGC7faurn+J/Mv90/W+iTtN0405/nLdopfMWwchyxhbGYl6TC4Sccn9TUkGAgg==",
"requires": {
"@types/color-name": "^1.1.1",
"color-convert": "^2.0.1"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -2171,9 +2116,9 @@
}
},
"spdx-correct": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz",
"integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==",
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz",
"integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==",
"dev": true,
"requires": {
"spdx-expression-parse": "^3.0.0",
@@ -2187,9 +2132,9 @@
"dev": true
},
"spdx-expression-parse": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
"integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz",
"integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==",
"dev": true,
"requires": {
"spdx-exceptions": "^2.1.0",
@@ -2219,7 +2164,6 @@
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
"integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
"dev": true,
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
@@ -2229,26 +2173,22 @@
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
"dev": true
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
"dev": true
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"dev": true
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"dev": true,
"requires": {
"ansi-regex": "^5.0.0"
}
@@ -2336,9 +2276,11 @@
"optional": true
},
"type-fest": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz",
"integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ=="
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
"integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
"dev": true,
"optional": true
},
"typedarray": {
"version": "0.0.6",
@@ -2371,12 +2313,12 @@
"dev": true
},
"unused-filename": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unused-filename/-/unused-filename-1.0.0.tgz",
"integrity": "sha1-00CID3GuIRXrqhMlvvBcxmhEacY=",
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/unused-filename/-/unused-filename-2.1.0.tgz",
"integrity": "sha512-BMiNwJbuWmqCpAM1FqxCTD7lXF97AvfQC8Kr/DIeA6VtvhJaMDupZ82+inbjl5yVP44PcxOuCSxye1QMS0wZyg==",
"requires": {
"modify-filename": "^1.1.0",
"path-exists": "^3.0.0"
"path-exists": "^4.0.0"
}
},
"update-notifier": {
@@ -2409,15 +2351,6 @@
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"pupa": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/pupa/-/pupa-2.0.1.tgz",
"integrity": "sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA==",
"dev": true,
"requires": {
"escape-goat": "^2.0.0"
}
}
}
},
@@ -2451,9 +2384,9 @@
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"v8-compile-cache": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz",
"integrity": "sha512-usZBT3PW+LOjM25wbqIlZwPeJV+3OSz3M1k1Ws8snlW39dZyYL9lOGC5FgPVHfk0jKmjiDV8Z0mIbVQPiwFs7g=="
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz",
"integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ=="
},
"validate-npm-package-license": {
"version": "3.0.4",
@@ -2538,15 +2471,15 @@
"dev": true
},
"yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true
},
"yargs": {
"version": "15.3.1",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz",
"integrity": "sha512-92O1HWEjw27sBfgmXiixJWT5hRBp2eobqXicLtPBIDBhYB+1HpwZlXmbW2luivBJHBzki+7VyCLRtAkScbTBQA==",
"version": "15.4.1",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
"integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
"dev": true,
"requires": {
"cliui": "^6.0.0",
@@ -2559,7 +2492,7 @@
"string-width": "^4.2.0",
"which-module": "^2.0.0",
"y18n": "^4.0.0",
"yargs-parser": "^18.1.1"
"yargs-parser": "^18.1.2"
}
},
"yargs-parser": {
+12 -17
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "2.11.7",
"version": "3.0.4",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -40,14 +40,9 @@
"win": {
"icon": "icon.ico",
"target": [
"nsis"
"dir"
]
},
"nsis": {
"createDesktopShortcut": "always",
"oneClick": false,
"perMachine": false
},
"linux": {
"synopsis": "vipergirls.to ripper",
"category": "Utility",
@@ -60,7 +55,7 @@
"mac": {
"category": "public.app-category.utilities",
"target": [
"dmg"
"dir"
],
"icon": "icon.icns"
}
@@ -70,16 +65,16 @@
"dist": "node pre-build.js && electron-builder"
},
"devDependencies": {
"electron": "8.2.4",
"electron-builder": "22.6.0"
"electron": "9.1.0",
"electron-builder": "22.7.0"
},
"dependencies": {
"axios": "^0.19.0",
"cheerio": "^1.0.0-rc.3",
"copy-dir": "^1.1.0",
"electron-context-menu": "^0.13.0",
"get-port": "^5.0.0",
"rimraf": "^2.6.3",
"v8-compile-cache": "^2.1.0"
"axios": "0.19.2",
"cheerio": "1.0.0-rc.3",
"copy-dir": "1.3.0",
"electron-context-menu": "2.1.0",
"get-port": "5.1.1",
"rimraf": "3.0.2",
"v8-compile-cache": "2.1.1"
}
}
+1 -1
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>2.11.7</version>
<version>3.0.4</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
+15 -6
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>2.11.7</version>
<version>3.0.4</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
@@ -32,6 +32,20 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.5.1</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -72,11 +86,6 @@
<artifactId>guava</artifactId>
<version>28.1-jre</version>
</dependency>
<dependency>
<groupId>org.imgscalr</groupId>
<artifactId>imgscalr-lib</artifactId>
<version>4.2</version>
</dependency>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-ui</artifactId>
@@ -5,20 +5,32 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.services.PersistenceService;
import tn.mnlr.vripper.jpa.repositories.IRepository;
import tn.mnlr.vripper.services.DataService;
import java.util.Set;
@Component
public class EventListenerBean {
@Autowired
private PersistenceService persistenceService;
@Getter
private static boolean init = false;
private final DataService dataService;
private final Set<IRepository> repositorySet;
@Autowired
public EventListenerBean(DataService dataService, Set<IRepository> repositorySet) {
this.dataService = dataService;
this.repositorySet = repositorySet;
}
@EventListener
public void onApplicationEvent(ContextRefreshedEvent event) {
repositorySet.forEach(IRepository::init);
dataService.setDownloadingToStopped();
init = true;
persistenceService.restore();
}
}
@@ -1,7 +1,6 @@
package tn.mnlr.vripper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@@ -11,10 +10,9 @@ import org.springframework.stereotype.Component;
import java.util.Map;
@Component
@Slf4j
public class SpringContext implements ApplicationContextAware {
private static final Logger logger = LoggerFactory.getLogger(SpringContext.class);
private static ConfigurableApplicationContext context;
public static <T> T getBean(Class<T> beanClass) {
@@ -26,7 +24,7 @@ public class SpringContext implements ApplicationContextAware {
}
public static void close() {
logger.info("Application terminating...");
log.info("Application terminating...");
if (context != null) {
context.close();
}
@@ -1,30 +1,27 @@
package tn.mnlr.vripper;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.RetryPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.time.temporal.ChronoUnit;
@SpringBootApplication
@Slf4j
public class VripperApplication {
private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
public static final RetryPolicy<Object> retryPolicy = new RetryPolicy<>()
.handleIf(e -> !(e instanceof InterruptedException))
.withDelay(1, 3, ChronoUnit.SECONDS)
.withMaxAttempts(5)
.abortOn(InterruptedException.class)
.onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
.onFailedAttempt(e -> log.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
public static void main(String[] args) {
try {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
SpringApplication.run(VripperApplication.class, args);
} catch (Exception e) {
logger.error("Failed to run the application", e);
log.error("Failed to run the application", e);
}
}
}
@@ -1,124 +0,0 @@
package tn.mnlr.vripper.entities;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import lombok.Getter;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
@Getter
public class Image {
private static final Logger logger = LoggerFactory.getLogger(Image.class);
private final AppStateService appStateService;
private final String type = "img";
private String postId;
private String postName;
private Host host;
private String url;
private int index;
private AtomicLong current = new AtomicLong(0);
@Setter
private long total = 0;
private Status status;
private BehaviorProcessor<Image> imageStateProcessor;
private Disposable subscription;
private Image() {
this.appStateService = SpringContext.getBean(AppStateService.class);
}
public Image(String url, String postId, String postName, Host host, int index) throws PostParseException {
this();
this.url = url;
this.postId = postId;
this.postName = postName;
this.host = host;
this.index = index;
status = Status.STOPPED;
if (!appStateService.newImage(this)) {
throw new PostParseException("Image already loaded");
}
}
public void setStatus(Status status) {
this.status = status;
update();
}
public boolean isCompleted() {
return status.equals(Status.COMPLETE);
}
public void init() {
cleanup();
imageStateProcessor = BehaviorProcessor.create();
subscription = imageStateProcessor
.onBackpressureBuffer()
.doOnNext(appStateService::imageUpdated)
.subscribe();
current.set(0);
status = Status.STOPPED;
imageStateProcessor.onNext(this);
}
public void cleanup() {
if (imageStateProcessor != null) {
imageStateProcessor.onComplete();
}
if (subscription != null) {
subscription.dispose();
}
}
public void setCurrent(int current) {
this.current.set(current);
update();
}
public void increase(int read) {
current.addAndGet(read);
update();
}
private void update() {
if (imageStateProcessor == null) {
return;
}
imageStateProcessor.onNext(this);
if (isCompleted()) {
imageStateProcessor.onComplete();
subscription.dispose();
imageStateProcessor = null;
}
}
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return url.equals(image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -1,124 +0,0 @@
package tn.mnlr.vripper.entities;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Getter
@ToString
public class Post {
public enum METADATA {
PREVIEWS, RESOLVED_NAME, POSTED_BY, THANKED
}
private final AppStateService appStateService;
private Status status;
private final String type = "post";
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private List<Image> images;
private Map<String, Object> metadata;
private AtomicInteger done = new AtomicInteger(0);
private int total;
private Set<String> hosts;
private boolean removed = false;
private String forum;
@Setter
private String postFolderName;
private Post() {
this.appStateService = SpringContext.getBean(AppStateService.class);
}
public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) throws PostParseException {
this();
this.title = title;
this.url = url;
this.images = images;
this.metadata = metadata;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
this.hosts = this.images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet());
total = images.size();
status = Status.STOPPED;
if (!this.appStateService.newPost(this)) {
throw new PostParseException("Post already loaded");
}
}
public void setTitle(String title) {
this.title = title;
updateNotification();
}
public void setRemoved(boolean removed) {
this.removed = removed;
updateNotification();
}
public void increase() {
done.incrementAndGet();
updateNotification();
}
public void setStatus(Status status) {
this.status = status;
updateNotification();
}
private void updateNotification() {
if (appStateService != null) {
appStateService.postUpdated(this);
}
}
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, PARTIAL, STOPPED
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return postId.equals(post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -1,12 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.databind.util.StdConverter;
import tn.mnlr.vripper.host.Host;
public class HostToString extends StdConverter<Host, String> {
@Override
public String convert(Host value) {
return value.getClass().getSimpleName();
}
}
@@ -1,36 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import lombok.Getter;
import lombok.Setter;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
@Getter
@Setter
public abstract class ImagePersistanceMixin {
@JsonSerialize(converter = HostToString.class)
@JsonDeserialize(converter = StringToHost.class)
private Host host;
@JsonIgnore
private BehaviorProcessor<Image> imageStateProcessor;
@JsonIgnore
private Disposable subscription;
@JsonIgnore
private String type;
@JsonIgnore
public abstract boolean isCompleted();
@JsonIgnore
private AppStateService appStateService;
}
@@ -1,21 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.Setter;
import tn.mnlr.vripper.services.AppStateService;
@Getter
@Setter
public abstract class PostPersistanceMixin {
@JsonIgnore
private String type;
@JsonIgnore
private AppStateService appStateService;
@JsonIgnore
private boolean removed;
}
@@ -1,17 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.databind.util.StdConverter;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import java.util.Collection;
public class StringToHost extends StdConverter<String, Host> {
private Collection<Host> hosts = SpringContext.getBeansOfType(Host.class).values();
@Override
public Host convert(String value) {
return hosts.stream().filter(e -> e.getClass().getSimpleName().equals(value)).findAny().orElse(null);
}
}
@@ -1,27 +0,0 @@
package tn.mnlr.vripper.entities.mixin.ui;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import lombok.Getter;
import lombok.Setter;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
@Getter
@Setter
public abstract class ImageUIMixin {
@JsonIgnore
private Host host;
@JsonIgnore
private BehaviorProcessor<Image> imageStateProcessor;
@JsonIgnore
private Disposable subscription;
@JsonIgnore
private AppStateService appStateService;
}
@@ -1,20 +0,0 @@
package tn.mnlr.vripper.entities.mixin.ui;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.Setter;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
@Getter
@Setter
public abstract class PostUIMixin {
@JsonIgnore
private List<Image> images;
@JsonIgnore
private AppStateService appStateService;
}
@@ -0,0 +1,16 @@
package tn.mnlr.vripper.exception;
public class QueueException extends Exception {
public QueueException(String message) {
super(message);
}
public QueueException(String message, Throwable e) {
super(message, e);
}
public QueueException(Throwable e) {
super(e);
}
}
@@ -1,5 +1,6 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -8,8 +9,6 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -23,10 +22,9 @@ import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class AcidimgHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(AcidimgHost.class);
private static final String host = "acidimg.cc";
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
private static final String IMG_XPATH = "//img[@class='centred']";
@@ -55,14 +53,14 @@ public class AcidimgHost extends Host {
Node contDiv;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.debug(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
httpPost.addHeader("Referer", url);
@@ -74,9 +72,9 @@ public class AcidimgHost extends Host {
throw new HostException(e);
}
logger.debug(String.format("Requesting %s", httpPost));
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
logger.debug(String.format("Cleaning response for %s", httpPost));
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
@@ -86,7 +84,7 @@ public class AcidimgHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
@@ -97,7 +95,7 @@ public class AcidimgHost extends Host {
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,6 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
@@ -8,16 +9,16 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.q.DownloadJob;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.*;
@@ -36,12 +37,13 @@ import java.util.Iterator;
import java.util.Objects;
@Service
@Slf4j
abstract public class Host {
private static final Logger logger = LoggerFactory.getLogger(Host.class);
private static final int READ_BUFFER_SIZE = 8192;
private static final Byte LOCK = 0;
@Autowired
protected HtmlProcessorService htmlProcessorService;
@@ -52,7 +54,7 @@ abstract public class Host {
private AppSettingsService appSettingsService;
@Autowired
private AppStateService appStateService;
private DataService dataService;
@Autowired
private ConnectionManager cm;
@@ -63,9 +65,6 @@ abstract public class Host {
@Autowired
private PathService pathService;
@Autowired
private AppStateExchange appStateExchange;
@Autowired
private VipergirlsAuthService authService;
@@ -81,78 +80,109 @@ abstract public class Host {
return url.contains(getLookup());
}
public void download(final Post post, final Image image, final ImageFileData imageFileData) throws DownloadException, InterruptedException {
public void download(final Post post, final Image image, final ImageFileData imageFileData, DownloadJob downloadJob) throws DownloadException, InterruptedException {
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
try {
File destinationFolder;
synchronized (appStateExchange.getPost(post.getPostId())) {
if (post.getPostFolderName() == null) {
pathService.createDefaultPostFolder(post);
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
HttpClientContext context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
destinationFolder = pathService.getDownloadDestinationFolder(post);
authService.leaveThanks(post);
}
appStateService.postDownloadingUpdate(image.getPostId());
imageFileData.setPageUrl(image.getUrl());
/*
* HOST SPECIFIC
*/
logger.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
setNameAndUrl(image.getUrl(), imageFileData, context);
logger.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
logger.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
logger.debug(String.format("Building image request for %s", image.getUrl()));
log.debug(String.format("Building image request for %s", image.getUrl()));
setImageRequest(imageFileData);
/*
* END HOST SPECIFIC
*/
String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
logger.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
log.debug(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
imageFileData.setImageName(formatImageFileName);
logger.debug(String.format("Saving to %s", destinationFolder.getPath()));
HttpClient client = cm.getClient().build();
logger.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
log.debug(String.format("Downloading %s", imageFileData.getImageUrl()));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest(), context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (downloadJob.isStopped()) {
return;
}
File destinationFolder;
synchronized (LOCK) {
Post updatedPost = dataService.findPostById(post.getId()).orElseThrow();
if (updatedPost.getPostFolderName() == null) {
pathService.createDefaultPostFolder(updatedPost);
}
destinationFolder = pathService.getDownloadDestinationFolder(updatedPost);
authService.leaveThanks(updatedPost);
}
File outputFile = new File(destinationFolder.getPath() + File.separator + String.format("%03d_", image.getIndex()) + imageFileData.getImageName() + ".tmp");
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
if (downloadJob.isStopped()) {
return;
}
image.setTotal(response.getEntity().getContentLength());
logger.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
logger.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !downloadJob.isStopped()) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
if (downloadJob.isStopped()) {
return;
}
}
File finalName = checkImageTypeAndRename(post, outputFile, imageFileData.getImageName(), image.getIndex());
File finalName = checkImageTypeAndRename(dataService.findPostById(post.getId()).orElseThrow(), outputFile, imageFileData.getImageName(), image.getIndex());
imageFileData.setFileName(finalName.getName());
}
} catch (Exception e) {
if (Thread.interrupted()) {
throw new InterruptedException("Download was interrupted");
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal()) {
image.setStatus(Status.COMPLETE);
} else if (downloadJob.isStopped()) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
downloadJob.done();
}
}
@@ -199,7 +229,7 @@ abstract public class Host {
File downloadDestinationFolder = pathService.getDownloadDestinationFolder(post);
File outImage = new File(downloadDestinationFolder, (appSettingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
if (outImage.exists() && outImage.delete()) {
logger.debug(String.format("%s is deleted", outImage.toString()));
log.debug(String.format("%s is deleted", outImage.toString()));
}
return Files.move(outputFile.toPath(), outImage.toPath(), StandardCopyOption.ATOMIC_MOVE).toFile();
} catch (Exception e) {
@@ -209,7 +239,7 @@ abstract public class Host {
final String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
@@ -219,21 +249,21 @@ abstract public class Host {
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
Header[] headers;
logger.debug(String.format("Requesting %s", url));
log.debug(String.format("Requesting %s", url));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
logger.debug(String.format("%s response: %n%s", url, basePage));
log.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Cleaning %s response", url));
log.debug(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
@@ -283,8 +313,8 @@ abstract public class Host {
this.headers = headers;
}
private Document document;
private Header[] headers;
private final Document document;
private final Header[] headers;
}
@Override
@@ -1,12 +1,11 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -20,10 +19,9 @@ import tn.mnlr.vripper.services.ConnectionManager;
import java.io.IOException;
@Service
@Slf4j
public class ImageBamHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
private static final String host = "imagebam.com";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
private static final String IMG_XPATH = "//img[@class='image']";
@@ -53,22 +51,22 @@ public class ImageBamHost extends Host {
Node contDiv;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.debug(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
httpGet.addHeader("Referer", url);
logger.debug(String.format("Requesting %s", httpGet));
log.debug(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
String s = EntityUtils.toString(res.getEntity());
logger.debug(String.format("%s response is:%n%s", httpGet, s));
logger.debug(String.format("Cleaning response for %s", httpGet));
log.debug(String.format("%s response is:%n%s", httpGet, s));
log.debug(String.format("Cleaning response for %s", httpGet));
doc = htmlProcessorService.clean(s);
EntityUtils.consumeQuietly(res.getEntity());
} catch (IOException | HtmlProcessorException e) {
@@ -78,14 +76,14 @@ public class ImageBamHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -13,10 +12,10 @@ import tn.mnlr.vripper.q.ImageFileData;
import java.util.Optional;
@Service
@Slf4j
public class ImageTwistHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
private static final String host = "imagetwist.com";
public ImageTwistHost() {
@@ -40,14 +39,14 @@ public class ImageTwistHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -10,16 +9,13 @@ import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import java.net.URI;
@Service
@Slf4j
public class ImageVenueHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageVenueHost.class);
private static final String host = "imagevenue.com";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
private static final String IMG_XPATH = "//img[@id='thepic']";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
public ImageVenueHost() {
super();
@@ -36,18 +32,16 @@ public class ImageVenueHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData, final HttpClientContext context) throws HostException {
//Sadly, they do not support https.
//If they add such support in the future,
//then we should automatically adapt the URL here as done elsewhere.
String url = _url.replace("http://", "https://");
Response resp = getResponse(url, context);
Document doc = resp.getDocument();
try {
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if(xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
//Button detected. No need to actually click it, just make the call again.
resp = getResponse(url, context);
doc = resp.getDocument();
@@ -58,7 +52,7 @@ public class ImageVenueHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
@@ -69,14 +63,13 @@ public class ImageVenueHost extends Host {
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
URI baseUri = new URI(url);
imageFileData.setImageUrl(new URI(baseUri.getScheme(), baseUri.getHost(), '/'+imgUrl, null).toString());
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
} catch(Exception e) {
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class ImageZillaHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
private static final String host = "imagezilla.net";
private static final String lookup = "imagezilla.net/show";
private static final String IMG_XPATH = "//img[@id='photo']";
@@ -40,9 +38,9 @@ public class ImageZillaHost extends Host {
String title;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
logger.debug(String.format("Resolving name for %s", url));
log.debug(String.format("Resolving name for %s", url));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class ImgSpiceHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImgSpiceHost.class);
private static final String host = "imgspice.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
@@ -41,14 +39,14 @@ public class ImgSpiceHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -13,10 +12,9 @@ import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
@Service
@Slf4j
public class ImgboxHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
private static final String host = "imgbox.com";
private static final String IMG_XPATH = "//img[@id='img']";
@@ -44,14 +42,14 @@ public class ImgboxHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,5 +1,6 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -8,8 +9,6 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -25,10 +24,9 @@ import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class ImxHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
private static final String host = "imx.to";
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
private static final String IMG_XPATH = "//img[@class='centred']";
@@ -56,7 +54,7 @@ public class ImxHost extends Host {
Node contDiv;
String value = null;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
Node node = contDiv.getAttributes().getNamedItem("value");
if (node != null) {
@@ -69,7 +67,7 @@ public class ImxHost extends Host {
if (value == null) {
throw new HostException("Failed to obtain value attribute from continue input");
}
logger.debug(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
List<NameValuePair> params = new ArrayList<>();
@@ -79,9 +77,9 @@ public class ImxHost extends Host {
} catch (Exception e) {
throw new HostException(e);
}
logger.debug(String.format("Requesting %s", httpPost));
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
logger.debug(String.format("Cleaning response for %s", httpPost));
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
@@ -90,14 +88,14 @@ public class ImxHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class PimpandhostHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PimpandhostHost.class);
private static final String host = "pimpandhost.com";
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
@@ -47,14 +45,14 @@ public class PimpandhostHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class PixRouteHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PixRouteHost.class);
private static final String host = "pixroute.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
@@ -36,7 +34,7 @@ public class PixRouteHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
@@ -47,7 +45,7 @@ public class PixRouteHost extends Host {
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
imageFileData.setImageUrl(imgNode.getAttributes().getNamedItem("src").getTextContent().trim());
imageFileData.setImageName(imgNode.getAttributes().getNamedItem("alt").getTextContent().trim());
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class PixhostHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
private static final String host = "pixhost.to";
private static final String lookup = "pixhost.to/show";
private static final String IMG_XPATH = "//img[@id='image']";
@@ -40,14 +38,14 @@ public class PixhostHost extends Host {
Node imgNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class PixxxelsHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PixxxelsHost.class);
private static final String host = "pixxxels.cc";
private static final String IMG_XPATH = "//*[@id='download']";
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
@@ -42,10 +40,10 @@ public class PixxxelsHost extends Host {
Node imgNode, titleNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
} catch (XpathException e) {
@@ -53,7 +51,7 @@ public class PixxxelsHost extends Host {
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = titleNode.getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -13,10 +12,9 @@ import tn.mnlr.vripper.q.ImageFileData;
import java.util.Optional;
@Service
@Slf4j
public class PostImgHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PostImgHost.class);
private static final String host = "postimg.cc";
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
private static final String IMG_XPATH = "//a[@id='download']";
@@ -43,17 +41,17 @@ public class PostImgHost extends Host {
Node urlNode, titleNode;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
logger.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.debug(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> getDefaultImageName(url));
imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
@@ -1,8 +1,7 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -11,10 +10,9 @@ import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
@Service
@Slf4j
public class TurboImageHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
private static final String host = "turboimagehost.com";
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
private static final String IMG_XPATH = "//img[@id='uImage']";
@@ -40,9 +38,9 @@ public class TurboImageHost extends Host {
String title;
try {
logger.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
logger.debug(String.format("Resolving name for %s", url));
log.debug(String.format("Resolving name for %s", url));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
@@ -0,0 +1,53 @@
package tn.mnlr.vripper.jpa;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
@EnableScheduling
public class Management {
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Pattern BACKUP_FILE_PATTERN = Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
private final JdbcTemplate jdbcTemplate;
private final String backupFolder;
@Autowired
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
this.jdbcTemplate = jdbcTemplate;
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
}
@PostConstruct
@Scheduled(cron = "0 0 0 ? * *")
private void backup() {
for (File file : Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[]{})) {
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
if (matcher.find()) {
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
if (localDate.isEqual(LocalDate.now())) {
return;
}
}
}
// create a backup file
File backupFile = new File(backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
}
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Objects;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Image {
@JsonIgnore
protected Long id;
@JsonIgnore
private Host host;
private String url;
private int index;
private long current = 0;
private long total = 0;
private Status status;
private String postId;
@JsonIgnore
private Long postIdRef;
public Image(String postId, String url, Host host, int index) {
this.postId = postId;
this.url = url;
this.host = host;
this.index = index;
status = Status.STOPPED;
}
public void increase(int read) {
current += read;
}
public void init() {
current = 0;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return Objects.equals(url, image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -0,0 +1,24 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.Collections;
import java.util.List;
@Getter
@Setter
@NoArgsConstructor
public class Metadata {
@JsonIgnore
private Long id;
private List<String> resolvedNames = Collections.emptyList();
private String postedBy;
private Long postIdRef;
}
@@ -0,0 +1,80 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Post {
@JsonIgnore
private Long id;
private Status status;
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private int done;
private int total;
private Set<String> hosts;
private String forum;
@JsonIgnore
private String securityToken;
@JsonIgnore
private String postFolderName;
private boolean thanked;
private Set<String> previews = Collections.emptySet();
private Metadata metadata;
private boolean renaming;
public Post(String title, String url, String postId, String threadId, String threadTitle, String forum, String securityToken) {
this.title = title;
this.url = url;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return Objects.equals(postId, post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -0,0 +1,57 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.util.Objects;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class Queued {
@JsonIgnore
private Long id;
private String link;
private String threadId;
private String postId;
private int total = 0;
private boolean loading = true;
public Queued(String link, String threadId, String postId) {
this();
this.link = link;
this.threadId = threadId;
this.postId = postId;
}
public void done() {
this.loading = false;
}
public void increment() {
this.total++;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Queued that = (Queued) o;
return Objects.equals(threadId, that.threadId);
}
@Override
public int hashCode() {
return Objects.hash(threadId);
}
}
@@ -0,0 +1,5 @@
package tn.mnlr.vripper.jpa.domain.enums;
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED, PARTIAL
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.List;
import java.util.Optional;
public interface IImageRepository extends IRepository {
Image save(Image image);
void deleteAllByPostId(String postId);
List<Image> findByPostId(String postId);
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
int stopByPostIdAndIsNotCompleted(String postId);
List<Image> findByPostIdAndIsError(String postId);
Optional<Image> findById(Long id);
int updateStatus(Status status, Long id);
int updateCurrent(long current, Long id);
int updateTotal(long total, Long id);
}
@@ -0,0 +1,16 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Metadata;
import java.util.Optional;
public interface IMetadataRepository extends IRepository {
Metadata save(Metadata metadata);
Optional<Metadata> findById(Long id);
Optional<Metadata> findByPostId(String postId);
int deleteByPostId(String postId);
}
@@ -0,0 +1,38 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.List;
import java.util.Optional;
public interface IPostRepository extends IRepository {
Post save(Post post);
int delete(Long id);
Optional<Post> findByPostId(String postId);
List<String> findCompleted();
Optional<Post> findById(Long id);
List<Post> findAll();
boolean existByPostId(String postId);
int setDownloadingToStopped();
int deleteByPostId(String postId);
int updateStatus(Status status, Long id);
int updateDone(int done, Long id);
int updateFolderName(String postFolderName, Long id);
int updateTitle(String title, Long id);
int updateThanked(boolean thanked, Long id);
}
@@ -0,0 +1,18 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.util.List;
import java.util.Optional;
public interface IQueuedRepository extends IRepository {
Queued save(Queued queued);
Optional<Queued> findByThreadId(String threadId);
List<Queued> findAll();
Optional<Queued> findById(Long id);
int deleteByThreadId(String threadId);
}
@@ -0,0 +1,6 @@
package tn.mnlr.vripper.jpa.repositories;
public interface IRepository {
public void init();
}
@@ -0,0 +1,141 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class ImageRepository implements IImageRepository {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
@Autowired
public ImageRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM IMAGE",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Image save(Image image) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
id,
image.getCurrent(),
image.getHost().getHost(),
image.getIndex(),
image.getPostId(),
image.getStatus().name(),
image.getTotal(),
image.getUrl(),
image.getPostIdRef()
);
image.setId(id);
return image;
}
@Override
public void deleteAllByPostId(String postId) {
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
}
@Override
public List<Image> findByPostId(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE WHERE POST_ID = ?",
new Object[]{postId},
new ImageRowMapper()
);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'",
Integer.class
);
}
@Override
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
new Object[]{postId},
new ImageRowMapper()
);
}
@Override
public int stopByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
postId
);
}
@Override
public List<Image> findByPostIdAndIsError(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
new Object[]{postId},
new ImageRowMapper()
);
}
@Override
public Optional<Image> findById(Long id) {
List<Image> images = jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.ID = ?",
new Object[]{id},
new ImageRowMapper()
);
if (images.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(images.get(0));
}
}
@Override
public int updateStatus(Status status, Long id) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?",
status.name(), id
);
}
@Override
public int updateCurrent(long current, Long id) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?",
current, id
);
}
@Override
public int updateTotal(long total, Long id) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?",
total, id
);
}
}
@@ -0,0 +1,29 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.sql.ResultSet;
import java.sql.SQLException;
public class ImageRowMapper implements RowMapper<Image> {
@Override
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
Image image = new Image();
image.setId(rs.getLong("ID"));
String host = rs.getString("HOST");
image.setHost(SpringContext.getBeansOfType(Host.class).values().stream().filter(e -> e.getHost().equals(host)).findAny().orElse(null));
image.setUrl(rs.getString("URL"));
image.setIndex(rs.getInt("INDEX"));
image.setCurrent(rs.getLong("CURRENT"));
image.setTotal(rs.getLong("TOTAL"));
image.setStatus(Status.valueOf(rs.getString("STATUS")));
image.setPostId(rs.getString("POST_ID"));
image.setPostIdRef(rs.getLong("POST_ID_REF"));
return image;
}
}
@@ -0,0 +1,85 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class MetadataRepository implements IMetadataRepository {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
@Autowired
public MetadataRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM IMAGE",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Metadata save(Metadata metadata) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO METADATA (ID, POSTED_BY, RESOLVED_NAMES, POST_ID_REF) VALUES (?,?,?,?)",
id,
metadata.getPostedBy(),
String.join("%sep%", metadata.getResolvedNames()),
metadata.getPostIdRef()
);
metadata.setId(id);
return metadata;
}
@Override
public Optional<Metadata> findById(Long id) {
List<Metadata> metadata = jdbcTemplate.query(
"SELECT * FROM METADATA AS metadata WHERE metadata.ID = ?",
new Object[]{id},
new MetadataRowMapper()
);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public Optional<Metadata> findByPostId(String postId) {
List<Metadata> metadata = jdbcTemplate.query(
"SELECT metadata.* FROM METADATA AS metadata INNER JOIN POST post ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?",
new Object[]{postId},
new MetadataRowMapper()
);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM METADATA WHERE POST_ID_REF = (SELECT post.ID FROM POST AS post INNER JOIN METADATA metadata ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?)",
postId
);
}
}
@@ -0,0 +1,24 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Metadata;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
public class MetadataRowMapper implements RowMapper<Metadata> {
@Override
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
Metadata metadata = new Metadata();
metadata.setId(rs.getLong("ID"));
metadata.setPostedBy(rs.getString("POSTED_BY"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
return metadata;
}
}
@@ -0,0 +1,183 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class PostRepository implements IPostRepository {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
@Autowired
public PostRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM POST",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Post save(Post post) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
String.join(";", post.getHosts()),
post.getPostFolderName(),
post.getPostId(),
String.join(";", post.getPreviews()),
post.getSecurityToken(),
post.getStatus().name(),
post.isThanked(),
post.getThreadId(),
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
post.getUrl()
);
post.setId(id);
return null;
}
@Override
public int delete(Long id) {
return jdbcTemplate.update(
"DELETE FROM POST AS post WHERE post.ID = ?",
id
);
}
@Override
public Optional<Post> findByPostId(String postId) {
List<Post> posts = jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE POST_ID = ?",
new Object[]{postId},
new PostRowMapper()
);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<String> findCompleted() {
return jdbcTemplate.query(
"SELECT POST_ID FROM POST WHERE status = 'COMPLETE' AND done >= total",
((rs, rowNum) -> rs.getString("POST_ID"))
);
}
@Override
public Optional<Post> findById(Long id) {
List<Post> posts = jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
new Object[]{id},
new PostRowMapper()
);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<Post> findAll() {
return jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
new PostRowMapper()
);
}
@Override
public boolean existByPostId(String postId) {
Integer count = jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?",
new Object[]{postId},
Integer.class
);
if (count == null) {
return false;
} else {
return count > 0;
}
}
@Override
public int setDownloadingToStopped() {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'"
);
}
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM POST AS post WHERE post.POST_ID = ?",
postId
);
}
@Override
public int updateStatus(Status status, Long id) {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?",
status.name(), id
);
}
@Override
public int updateDone(int done, Long id) {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?",
done, id
);
}
@Override
public int updateFolderName(String postFolderName, Long id) {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
postFolderName, id
);
}
@Override
public int updateTitle(String title, Long id) {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?",
title, id
);
}
@Override
public int updateThanked(boolean thanked, Long id) {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?",
thanked, id
);
}
}
@@ -0,0 +1,55 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Set;
public class PostRowMapper implements RowMapper<Post> {
private static final String DELIMITER = ";";
@Override
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
Post post = new Post();
post.setId(rs.getLong("post.ID"));
post.setStatus(Status.valueOf(rs.getString("STATUS")));
post.setPostId(rs.getString("POST_ID"));
post.setThreadTitle(rs.getString("THREAD_TITLE"));
post.setThreadId(rs.getString("THREAD_ID"));
post.setTitle(rs.getString("TITLE"));
post.setUrl(rs.getString("URL"));
post.setDone(rs.getInt("DONE"));
post.setTotal(rs.getInt("TOTAL"));
post.setHosts(Set.of(rs.getString("HOSTS").split(DELIMITER)));
post.setForum(rs.getString("FORUM"));
post.setSecurityToken(rs.getString("SECURITY_TOKEN"));
post.setPostFolderName(rs.getString("POST_FOLDER_NAME"));
post.setThanked(rs.getBoolean("THANKED"));
String previews;
if ((previews = rs.getString("PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
Long metadataId = rs.getLong("metadata.ID");
if (!rs.wasNull()) {
Metadata metadata = new Metadata();
metadata.setId(metadataId);
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostedBy(rs.getString("POSTED_BY"));
post.setMetadata(metadata);
}
return post;
}
}
@@ -0,0 +1,95 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class QueuedRepository implements IQueuedRepository {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
@Autowired
public QueuedRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM QUEUED",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Queued save(Queued queued) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
id,
queued.getTotal(),
queued.getLink(),
queued.isLoading(),
queued.getPostId(),
queued.getThreadId()
);
queued.setId(id);
return queued;
}
@Override
public Optional<Queued> findByThreadId(String threadId) {
List<Queued> queuedList = jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
new Object[]{threadId},
new QueuedRowMapper()
);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public List<Queued> findAll() {
return jdbcTemplate.query(
"SELECT * FROM QUEUED",
new QueuedRowMapper()
);
}
@Override
public Optional<Queued> findById(Long id) {
List<Queued> queuedList = jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?",
new Object[]{id},
new QueuedRowMapper()
);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public int deleteByThreadId(String threadId) {
return jdbcTemplate.update(
"DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?",
threadId
);
}
}
@@ -0,0 +1,22 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.sql.ResultSet;
import java.sql.SQLException;
public class QueuedRowMapper implements RowMapper<Queued> {
@Override
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
Queued queued = new Queued();
queued.setId(rs.getLong("ID"));
queued.setLink(rs.getString("LINK"));
queued.setThreadId(rs.getString("THREAD_ID"));
queued.setPostId(rs.getString("POST_ID"));
queued.setTotal(rs.getInt("TOTAL"));
queued.setLoading(rs.getBoolean("LOADING"));
return queued;
}
}
@@ -1,39 +1,44 @@
package tn.mnlr.vripper.q;
import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.function.CheckedRunnable;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import java.util.Objects;
import java.util.concurrent.Callable;
public class DownloadJob implements Callable<Image> {
private static final Logger logger = LoggerFactory.getLogger(DownloadJob.class);
@Slf4j
public class DownloadJob implements CheckedRunnable {
@Getter
private final Image image;
@Getter
private final Post post;
@Getter
private final ImageFileData imageFileData = new ImageFileData();
@Getter
private boolean stopped = false;
@Getter
private boolean finished = false;
DownloadJob(Post post, Image image) {
this.image = image;
this.post = post;
}
@Override
public Image call() throws Exception {
logger.debug(String.format("Starting downloading %s", image.getUrl()));
image.setStatus(Image.Status.DOWNLOADING);
image.setCurrent(0);
image.getHost().download(post, image, imageFileData);
image.setStatus(Image.Status.COMPLETE);
return image;
public void run() throws Exception {
if (stopped) {
done();
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
image.getHost().download(post, image, imageFileData, this);
}
@Override
@@ -41,11 +46,20 @@ public class DownloadJob implements Callable<Image> {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return image.equals(that.image);
return Objects.equals(image, that.image) &&
Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image);
return Objects.hash(image, post);
}
public void stop() {
this.stopped = true;
}
public void done() {
finished = true;
}
}
@@ -1,87 +0,0 @@
package tn.mnlr.vripper.q;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.AppStateService;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
@Service
public class DownloadQ {
private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
private final AppStateService appStateService;
private final AppSettingsService appSettingsService;
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> downloadQ = new ConcurrentHashMap<>();
@Autowired
public DownloadQ(AppStateService appStateService, AppSettingsService appSettingsService, List<Host> hosts) {
this.appStateService = appStateService;
this.appSettingsService = appSettingsService;
this.hosts = hosts;
}
@PostConstruct
private void init() {
hosts.forEach(host -> downloadQ.put(host, new LinkedBlockingDeque<>()));
}
public void put(Post post, Image image) throws InterruptedException {
logger.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
DownloadJob downloadJob = new DownloadJob(post, image);
downloadQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
appStateService.newDownloadJob(downloadJob);
}
public void remove(final DownloadJob downloadJob) {
downloadQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
}
public List<DownloadJob> peek() {
List<DownloadJob> downloadJobs = new ArrayList<>();
if (hosts.size() == 0) {
return downloadJobs;
}
for (Host host : hosts) {
Iterator<DownloadJob> it = downloadQ.get(host).iterator();
for (int i = 0; i < appSettingsService.getSettings().getMaxThreads(); i++) {
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
if (downloadJob != null) {
downloadJobs.add(downloadJob);
}
}
}
return downloadJobs;
}
public void enqueue(Post post) throws InterruptedException {
for (Image image : post.getImages()) {
put(post, image);
}
}
public int size() {
return downloadQ.values().stream().mapToInt(BlockingDeque::size).sum();
}
public Iterable<? extends Map.Entry<Host, BlockingDeque<DownloadJob>>> entries() {
return downloadQ.entrySet();
}
}
@@ -0,0 +1,47 @@
package tn.mnlr.vripper.q;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.MutexService;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public class ExecuteRunnable implements Runnable {
private final ExecutionService executionService;
private final DataService dataService;
private final MutexService mutexService;
private final DownloadJob downloadJob;
public ExecuteRunnable(final DownloadJob downloadJob) {
executionService = SpringContext.getBean(ExecutionService.class);
dataService = SpringContext.getBean(DataService.class);
mutexService = SpringContext.getBean(MutexService.class);
this.downloadJob = downloadJob;
}
@Override
public void run() {
mutexService.createPostLock(downloadJob.getPost().getPostId());
ReentrantLock mutex = mutexService.getPostLock(downloadJob.getPost().getPostId());
mutex.lock();
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> {
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
})
.onComplete(e -> {
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
executionService.afterJobFinish(downloadJob);
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
mutex.unlock();
}).run(downloadJob);
}
}
@@ -1,64 +1,53 @@
package tn.mnlr.vripper.q;
import lombok.NonNull;
import net.jodah.failsafe.Failsafe;
import net.jodah.failsafe.FailsafeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.MutexService;
import tn.mnlr.vripper.services.post.PostService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Service
@Slf4j
public class ExecutionService {
private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
private final DownloadQ downloadQ;
private final AppSettingsService settings;
private final AppStateService appStateService;
private final AppStateExchange appStateExchange;
private final ThumbnailGenerator thumbnailGenerator;
private final AppSettingsService appSettingsService;
private final CommonExecutor commonExecutor;
private static final List<Status> FINISHED = Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
private final int MAX_POOL_SIZE = 12;
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
private boolean notPauseQ = true;
private ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final PendingQ pendingQ;
private final AppSettingsService settings;
private final DataService dataService;
private final PostService postService;
private final MutexService mutexService;
private boolean pauseQ = false;
private Thread executionThread;
private BlockingQueue<DownloadJob> queue = new LinkedBlockingQueue<>();
private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
private Thread pollThread;
@Autowired
public ExecutionService(DownloadQ downloadQ, AppSettingsService settings, AppStateService appStateService, AppStateExchange appStateExchange, ThumbnailGenerator thumbnailGenerator, AppSettingsService appSettingsService, CommonExecutor commonExecutor) {
this.downloadQ = downloadQ;
public ExecutionService(PendingQ pendingQ, AppSettingsService settings, DataService dataService, PostService postService, MutexService mutexService) {
this.pendingQ = pendingQ;
this.settings = settings;
this.appStateService = appStateService;
this.appStateExchange = appStateExchange;
this.thumbnailGenerator = thumbnailGenerator;
this.appSettingsService = appSettingsService;
this.commonExecutor = commonExecutor;
this.dataService = dataService;
this.postService = postService;
this.mutexService = mutexService;
}
@PostConstruct
@@ -71,134 +60,99 @@ public class ExecutionService {
@PreDestroy
private void destroy() throws InterruptedException {
logger.info("Shutting down ExecutionService");
log.info("Shutting down ExecutionService");
executionThread.interrupt();
pollThread.interrupt();
executor.shutdown();
appStateExchange.getPosts().keySet().forEach(p -> {
logger.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p);
dataService.findAllPosts().forEach(p -> {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
executor.awaitTermination(10, TimeUnit.SECONDS);
}
private void stopRunning(@NonNull String postId) {
List<DownloadJob> data = running
.stream()
.filter(e -> e.getImage().getPostId().equals(postId))
.peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
.collect(Collectors.toList());
logger.debug(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
data.forEach(e -> {
futures.get(e.getImage().getUrl()).cancel(true);
if (e.getImageFileData().getImageRequest() != null) {
e.getImageFileData().getImageRequest().abort();
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = executing.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
e.getImage().cleanup();
});
}
public synchronized void stopAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::stop);
} else {
appStateExchange.getPosts().values().forEach(p -> this.stop(p.getPostId()));
}
}
public synchronized void restartAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::restart);
} else {
appStateExchange.getPosts().values().forEach(p -> this.restart(p.getPostId()));
}
}
private void restart(@NonNull String postId) {
if (appStateService.isRunning(postId)) {
logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
Post post = appStateExchange.getPost(postId);
List<Image> images = post
.getImages()
.stream()
.filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
.collect(Collectors.toList());
if (images.isEmpty()) {
return;
}
post.setStatus(Post.Status.PENDING);
logger.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
downloadQ.put(post, image);
Thread.sleep(200);
} catch (InterruptedException e) {
logger.warn("Thread was interrupted", e);
Thread.currentThread().interrupt();
}
}
}
private void removeScheduled(Image image) {
image.setStatus(Image.Status.STOPPED);
logger.debug(String.format("Removing scheduled job for %s", image.getUrl()));
boolean removed = false;
main:
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : downloadQ.entries()) {
Iterator<DownloadJob> iterator = entry.getValue().iterator();
while (iterator.hasNext()) {
DownloadJob next = iterator.next();
if (next.getImage().getPostId().equals(image.getPostId())) {
iterator.remove();
appStateService.doneDownloadJob(image);
logger.debug(String.format("Scheduled job for %s is removed", image.getUrl()));
removed = true;
break main;
}
}
public void stopAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::stop);
} else {
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
}
if (!removed) {
logger.debug(String.format("Job for %s does not exist", image.getUrl()));
}
image.cleanup();
}
private void removeRunning(String postId) {
logger.debug(String.format("Interrupting running jobs for post id %s", postId));
stopRunning(postId);
public void restartAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::restart);
} else {
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
}
}
private void restart(@NonNull String postId) {
if (isPending(postId)) {
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
return;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
post.setStatus(Status.PENDING);
dataService.updatePostStatus(post.getStatus(), post.getId());
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
try {
pendingQ.put(post, image);
} catch (InterruptedException e) {
log.warn("Thread was interrupted", e);
Thread.currentThread().interrupt();
}
}
}
private boolean isPending(String postId) {
return pendingQ.isPending(postId);
}
private void stop(String postId) {
try {
final Post post = appStateExchange.getPost(postId);
pauseQ = true;
final Post post = dataService.findPostByPostId(postId).orElseThrow();
if (post == null) {
return;
}
if (FINISHED.contains(post.getStatus())) {
return;
}
notPauseQ = false;
post.setStatus(Post.Status.STOPPED);
List<Image> images = post
.getImages()
.stream()
.filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
.collect(Collectors.toList());
if (images.isEmpty()) {
return;
}
logger.debug(String.format("Stopping %d jobs for post id %s", images.size(), postId));
images.forEach(this::removeScheduled);
removeRunning(postId);
pendingQ.stop(post);
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
postService.stopFetchingMetadata(post);
} finally {
notPauseQ = true;
pauseQ = false;
}
}
@@ -209,7 +163,7 @@ public class ExecutionService {
threadCount.put(host, new AtomicInteger(0));
}
canRun = threadCount.get(host).get() < settings.getSettings().getMaxThreads() && (settings.getSettings().getMaxTotalThreads() == 0 ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE : threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getSettings().getMaxTotalThreads());
if (canRun && notPauseQ) {
if (canRun && !pauseQ) {
threadCount.get(host).incrementAndGet();
return true;
}
@@ -219,11 +173,11 @@ public class ExecutionService {
private void poll() {
while (!Thread.interrupted()) {
try {
List<DownloadJob> peek = downloadQ.peek();
List<DownloadJob> peek = pendingQ.peek();
for (DownloadJob downloadJob : peek) {
if (canRun(downloadJob.getImage().getHost())) {
queue.offer(downloadJob);
downloadQ.remove(downloadJob);
executionQueue.offer(downloadJob);
pendingQ.remove(downloadJob);
}
}
synchronized (threadCount) {
@@ -239,52 +193,37 @@ public class ExecutionService {
private void start() {
while (!Thread.interrupted()) {
try {
push(queue.take());
push(executionQueue.take());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
} catch (Exception e) {
logger.error("Execution Service failed", e);
log.error("Execution Service failed", e);
break;
}
}
}
private void push(DownloadJob take) {
Runnable task = () -> {
running.add(take);
private void push(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new ExecuteRunnable(downloadJob));
executing.add(downloadJob);
}
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> {
if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
logger.debug("Job successfully interrupted");
return;
}
logger.error(String.format("Failed to download %s after %d tries", take.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
take.getImage().setStatus(Image.Status.ERROR);
})
.onComplete(e -> {
appStateService.doneDownloadJob(take.getImage());
logger.debug(String.format("Finished downloading %s", take.getImage().getUrl()));
if (appSettingsService.getSettings().getViewPhotos()) {
commonExecutor.getGeneralExecutor().submit(
() -> thumbnailGenerator.getThumbnails()
.get(new ThumbnailGenerator.CacheKey(take.getImage().getPostId(), take.getImageFileData().getFileName())));
}
threadCount.get(take.getImage().getHost()).decrementAndGet();
running.remove(take);
futures.remove(take.getImage().getUrl());
synchronized (threadCount) {
threadCount.notify();
}
})
.get(take::call);
};
logger.debug(String.format("Scheduling a job for %s", take.getImage().getUrl()));
futures.put(take.getImage().getUrl(), executor.submit(task));
public synchronized void afterJobFinish(DownloadJob downloadJob) {
int count = pendingQ.decrement(downloadJob.getPost().getPostId());
if (count == 0) {
dataService.finishPost(downloadJob.getPost());
mutexService.removePostLock(downloadJob.getPost().getPostId());
}
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
executing.remove(downloadJob);
synchronized (threadCount) {
threadCount.notify();
}
}
public int runningCount() {
return running.size();
return executing.size();
}
}
@@ -0,0 +1,115 @@
package tn.mnlr.vripper.q;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.DataService;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
@Service
@Slf4j
public class PendingQ {
private final DataService dataService;
private final AppSettingsService appSettingsService;
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
@Autowired
public PendingQ(DataService dataService, AppSettingsService appSettingsService, List<Host> hosts) {
this.dataService = dataService;
this.appSettingsService = appSettingsService;
this.hosts = hosts;
}
@PostConstruct
private void init() {
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
}
public void put(Post post, Image image) throws InterruptedException {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob = new DownloadJob(post, image);
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
checkKey(post.getPostId());
toBeExecuted.get(post.getPostId()).incrementAndGet();
}
private synchronized void checkKey(String postId) {
if (!toBeExecuted.containsKey(postId)) {
toBeExecuted.put(postId, new AtomicInteger(0));
}
}
public void remove(final DownloadJob downloadJob) {
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
}
public List<DownloadJob> peek() {
List<DownloadJob> downloadJobs = new ArrayList<>();
if (hosts.size() == 0) {
return downloadJobs;
}
for (Host host : hosts) {
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
for (int i = 0; i < appSettingsService.getSettings().getMaxThreads(); i++) {
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
if (downloadJob != null) {
downloadJobs.add(downloadJob);
}
}
}
return downloadJobs;
}
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
for (Image image : images) {
put(post, image);
}
}
public int size() {
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
}
public void stop(Post post) {
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
entry.getValue().removeIf(predicate);
decrement(post.getPostId());
}
}
public boolean isPending(String postId) {
return toBeExecuted.containsKey(postId);
}
public synchronized int decrement(String postId) {
AtomicInteger counter = toBeExecuted.get(postId);
if (counter == null) {
return 0;
}
int count = counter.decrementAndGet();
if (count == 0) {
toBeExecuted.remove(postId);
}
return count;
}
}
@@ -7,9 +7,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.SpringContext;
@@ -28,20 +27,20 @@ import static java.nio.file.StandardOpenOption.*;
@Service
@Getter
@Setter
@Slf4j
public class AppSettingsService {
private final String MAX_TOTAL_THREADS = "MAX_TOTAL_THREADS";
private final String baseDir;
private final Path configPath;
private final Logger logger = LoggerFactory.getLogger(AppSettingsService.class);
private final ObjectMapper om = new ObjectMapper();
private Settings settings = new Settings();
public AppSettingsService(@Value("${base.dir}") String baseDir) {
public AppSettingsService(@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
this.baseDir = baseDir;
this.configPath = Paths.get(baseDir, ".vripper", "config.json");
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
@@ -72,16 +71,16 @@ public class AppSettingsService {
try {
check(this.settings);
} catch (ValidationException e) {
logger.error(String.format("Your settings are invalid, either remove %s, or fix it", configPath.toString()), e);
log.error(String.format("Your settings are invalid, either remove %s, or fix it", configPath.toString()), e);
SpringContext.close();
}
} catch (IOException e) {
logger.error("Failed restore user settings", e);
log.error("Failed restore user settings", e);
settings = new Settings();
}
if (settings.getDownloadPath() == null) {
settings.setDownloadPath(baseDir);
settings.setDownloadPath(System.getProperty("user.home"));
}
if (settings.getMaxThreads() == null) {
@@ -140,11 +139,6 @@ public class AppSettingsService {
settings.setViewPhotos(false);
}
if (settings.getNotification() == null) {
settings.setNotification(false);
}
save();
}
@@ -156,7 +150,7 @@ public class AppSettingsService {
Files.write(configPath, om.writeValueAsBytes(settings), CREATE, WRITE, TRUNCATE_EXISTING, SYNC);
} catch (IOException e) {
logger.error("Failed to store user settings", e);
log.error("Failed to store user settings", e);
}
}
@@ -238,8 +232,6 @@ public class AppSettingsService {
private Boolean clearCompleted;
@JsonProperty("viewPhotos")
private Boolean viewPhotos;
@JsonProperty("notification")
private Boolean notification;
@JsonProperty("darkTheme")
private Boolean darkTheme;
@@ -1,73 +0,0 @@
package tn.mnlr.vripper.services;
import io.reactivex.processors.PublishProcessor;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@Service
public class AppStateExchange {
private Map<String, Image> currentImages = new ConcurrentHashMap<>();
private Map<String, Post> currentPosts = new ConcurrentHashMap<>();
private Map<String, AtomicInteger> runningPosts = new ConcurrentHashMap<>();
private Map<String, QueuedVGLink> grabQueue = new ConcurrentHashMap<>();
private PublishProcessor<QueuedVGLink> liveGrabQueue = PublishProcessor.create();
private PublishProcessor<Image> liveImageUpdates = PublishProcessor.create();
private PublishProcessor<Post> livePostsState = PublishProcessor.create();
public Map<String, AtomicInteger> running() {
return runningPosts;
}
public Map<String, Post> getPosts() {
return this.currentPosts;
}
public Post getPost(String postId) {
return currentPosts.get(postId);
}
public Map<String, Image> getImages() {
return currentImages;
}
public PublishProcessor<Image> liveImage() {
return liveImageUpdates;
}
public PublishProcessor<Post> livePost() {
return livePostsState;
}
public Map<String, QueuedVGLink> getQueue() {
return grabQueue;
}
public PublishProcessor<QueuedVGLink> liveQueue() {
return liveGrabQueue;
}
public synchronized void restore(Map<String, Post> read) {
currentPosts.clear();
currentPosts.putAll(read);
currentPosts.values().forEach(p -> {
if (Arrays.asList(Post.Status.DOWNLOADING, Post.Status.PARTIAL, Post.Status.PENDING).contains(p.getStatus())) {
p.setStatus(Post.Status.STOPPED);
}
});
currentImages.clear();
read.values().stream().flatMap(e -> e.getImages().stream()).forEach(e -> currentImages.put(e.getUrl(), e));
}
}
@@ -1,176 +0,0 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.q.DownloadJob;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Service
public class AppStateService {
private final PersistenceService persistenceService;
private final AppSettingsService appSettingsService;
private final AppStateExchange appStateExchange;
@Autowired
public AppStateService(PersistenceService persistenceService, AppSettingsService appSettingsService, AppStateExchange appStateExchange) {
this.appStateExchange = appStateExchange;
this.persistenceService = persistenceService;
this.appSettingsService = appSettingsService;
}
public synchronized void newQueueLink(QueuedVGLink queuedVGLink) {
appStateExchange.getQueue().put(queuedVGLink.getThreadId(), queuedVGLink);
appStateExchange.liveQueue().onNext(queuedVGLink);
}
public synchronized void queueLinkUpdated(QueuedVGLink queuedVGLink) {
appStateExchange.liveQueue().onNext(queuedVGLink);
}
public synchronized void removeQueueLink(String threadId) {
Optional.ofNullable(appStateExchange.getQueue().remove(threadId)).ifPresent(QueuedVGLink::remove);
}
public synchronized boolean newPost(@NonNull Post post) {
if (appStateExchange.getPosts().containsKey(post.getPostId())) {
return false;
} else {
appStateExchange.getPosts().put(post.getPostId(), post);
appStateExchange.livePost().onNext(post);
persistenceService.getProcessor().onNext(appStateExchange.getPosts());
return true;
}
}
public synchronized void postUpdated(@NonNull Post post) {
appStateExchange.livePost().onNext(post);
persistenceService.getProcessor().onNext(appStateExchange.getPosts());
}
public boolean newImage(Image image) {
if (appStateExchange.getImages().containsKey(image.getUrl())) {
return false;
} else {
appStateExchange.getImages().put(image.getUrl(), image);
appStateExchange.liveImage().onNext(image);
persistenceService.getProcessor().onNext(appStateExchange.getPosts());
return true;
}
}
public synchronized void imageUpdated(Image image) {
persistenceService.getProcessor().onNext(appStateExchange.getPosts());
appStateExchange.liveImage().onNext(image);
if (image.isCompleted()) {
Post postState = appStateExchange.getPosts().get(image.getPostId());
postState.increase();
}
}
public synchronized void newDownloadJob(@NonNull DownloadJob downloadJob) {
String postId = downloadJob.getImage().getPostId();
checkKeyRunningPosts(postId);
appStateExchange.running().get(postId).incrementAndGet();
}
public synchronized void postDownloadingUpdate(@NonNull String postId) {
Post post = appStateExchange.getPosts().get(postId);
if (!post.getStatus().equals(Post.Status.DOWNLOADING) && !post.getStatus().equals(Post.Status.PARTIAL)) {
post.setStatus(Post.Status.DOWNLOADING);
appStateExchange.livePost().onNext(post);
}
}
public synchronized void doneDownloadJob(@NonNull Image image) {
String postId = image.getPostId();
int i = appStateExchange.running().get(postId).decrementAndGet();
Post post = appStateExchange.getPosts().get(postId);
if (image.getStatus().equals(Image.Status.ERROR)) {
post.setStatus(Post.Status.PARTIAL);
}
if (i == 0) {
appStateExchange.running().remove(postId);
if (post.getImages().stream().map(Image::getStatus).anyMatch(e -> e.equals(Image.Status.ERROR))) {
post.setStatus(Post.Status.ERROR);
} else {
if (!Post.Status.STOPPED.equals(post.getStatus())) {
post.setStatus(Post.Status.COMPLETE);
if (appSettingsService.getSettings().getClearCompleted()) {
remove(image.getPostId());
}
}
}
}
}
public boolean isRunning(String postId) {
AtomicInteger runningCount = appStateExchange.running().get(postId);
return runningCount != null && runningCount.get() > 0;
}
public synchronized List<String> clearAll() {
return this.appStateExchange.getPosts()
.values()
.stream()
.filter(e -> e.getStatus().equals(Post.Status.COMPLETE) && e.getDone().get() >= e.getTotal())
.map(Post::getPostId)
.peek(this::remove)
.collect(Collectors.toList());
}
public synchronized List<String> removeAll(List<String> postIds) {
if (postIds != null && !postIds.isEmpty()) {
postIds.forEach(this::remove);
return postIds;
} else {
return this.appStateExchange.getPosts()
.values()
.stream()
.map(Post::getPostId)
.peek(this::remove)
.collect(Collectors.toList());
}
}
private void checkKeyRunningPosts(@NonNull String key) {
if (!appStateExchange.running().containsKey(key)) {
appStateExchange.running().put(key, new AtomicInteger(0));
}
}
private void remove(@NonNull String postId) {
final Post post = appStateExchange.getPosts().get(postId);
if (post == null) {
return;
}
post.setRemoved(true);
appStateExchange.running().remove(postId);
appStateExchange.getPosts().remove(postId);
appStateExchange.getImages().entrySet().removeIf(entry -> entry.getValue().getPostId().equals(postId));
persistenceService.getProcessor().onNext(appStateExchange.getPosts());
}
@Getter
public static class CachedThread {
private Map<String, Post> posts = new ConcurrentHashMap<>();
private AtomicInteger parsed = new AtomicInteger(0);
@Setter
private int total = 0;
}
}
@@ -0,0 +1,260 @@
package tn.mnlr.vripper.services;
import io.reactivex.Observable;
import io.reactivex.processors.PublishProcessor;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@Service
@Transactional
@Slf4j
public class DataService {
private final IPostRepository postRepository;
private final IImageRepository imageRepository;
private final IQueuedRepository queuedRepository;
private final IMetadataRepository metadataRepository;
private final AppSettingsService appSettingsService;
private final PublishProcessor<Long> liveGrabQueue = PublishProcessor.create();
private final PublishProcessor<Long> liveImageUpdates = PublishProcessor.create();
private final PublishProcessor<Long> livePostsState = PublishProcessor.create();
private final PublishProcessor<String> postRemove = PublishProcessor.create();
private final PublishProcessor<String> queueRemove = PublishProcessor.create();
public Observable<Long> liveQueue() {
return liveGrabQueue.toObservable();
}
public Observable<Long> livePost() {
return livePostsState.toObservable();
}
public Observable<Long> liveImage() {
return liveImageUpdates.toObservable();
}
public Observable<String> queueRemove() {
return queueRemove.toObservable();
}
public Observable<String> postRemove() {
return postRemove.toObservable();
}
@Autowired
public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, AppSettingsService appSettingsService) {
this.postRepository = postRepository;
this.imageRepository = imageRepository;
this.queuedRepository = queuedRepository;
this.metadataRepository = metadataRepository;
this.appSettingsService = appSettingsService;
}
private void save(Post post) {
postRepository.save(post);
livePostsState.onNext(post.getId());
}
private void save(Queued queued) {
queuedRepository.save(queued);
liveGrabQueue.onNext(queued.getId());
}
private void save(Image image) {
imageRepository.save(image);
liveImageUpdates.onNext(image.getId());
}
public boolean exists(String postId) {
return postRepository.existByPostId(postId);
}
public void newPost(Post post, Collection<Image> images) {
save(post);
images.forEach(image -> {
image.setPostIdRef(post.getId());
save(image);
});
}
public void setDownloadingToStopped() {
postRepository.setDownloadingToStopped();
}
public synchronized void afterJobFinish(Image image, Post post) {
if (image.getStatus().equals(Status.COMPLETE)) {
post.setDone(post.getDone() + 1);
updatePostDone(post.getDone(), post.getId());
} else if (image.getStatus().equals(Status.ERROR)) {
post.setStatus(Status.PARTIAL);
updatePostStatus(post.getStatus(), post.getId());
}
}
private void updatePostDone(int done, Long id) {
livePostsState.onNext(id);
postRepository.updateDone(done, id);
}
public void finishPost(@NonNull Post post) {
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
post.setStatus(Status.ERROR);
updatePostStatus(post.getStatus(), post.getId());
} else {
if (post.getDone() < post.getTotal()) {
post.setStatus(Status.STOPPED);
updatePostStatus(post.getStatus(), post.getId());
} else {
post.setStatus(Status.COMPLETE);
updatePostStatus(post.getStatus(), post.getId());
if (appSettingsService.getSettings().getClearCompleted()) {
remove(post.getPostId());
}
}
}
}
private void remove(@NonNull final String postId) {
imageRepository.deleteAllByPostId(postId);
metadataRepository.deleteByPostId(postId);
postRepository.deleteByPostId(postId);
postRemove.onNext(postId);
}
public void newQueueLink(@NonNull final Queued queued) {
save(queued);
}
public void removeQueueLink(@NonNull final String threadId) {
queuedRepository.deleteByThreadId(threadId);
queueRemove.onNext(threadId);
}
public List<String> clearCompleted() {
List<String> completed = postRepository.findCompleted();
completed.forEach(this::remove);
return completed;
}
public void removeAll(final List<String> postIds) {
if (postIds != null && !postIds.isEmpty()) {
for (String postId : postIds) {
remove(postId);
}
} else {
postRepository.findAll().forEach(p -> remove(p.getPostId()));
}
}
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
return imageRepository.findByPostIdAndIsNotCompleted(postId);
}
public long countErrorImages() {
return imageRepository.countError();
}
public List<Image> findImagesByPostId(String postId) {
return imageRepository.findByPostId(postId);
}
public Iterable<Post> findAllPosts() {
return postRepository.findAll();
}
public Optional<Post> findPostByPostId(String postId) {
return postRepository.findByPostId(postId);
}
public void stopImagesByPostIdAndIsNotCompleted(String postId) {
imageRepository.stopByPostIdAndIsNotCompleted(postId);
}
public Optional<Queued> findQueuedByThreadId(String threadId) {
return queuedRepository.findByThreadId(threadId);
}
public Iterable<Queued> findAllQueued() {
return queuedRepository.findAll();
}
public Optional<Post> findPostById(Long aLong) {
return postRepository.findById(aLong);
}
public Optional<Image> findImageById(Long aLong) {
return imageRepository.findById(aLong);
}
public Optional<Queued> findQueuedById(Long aLong) {
return queuedRepository.findById(aLong);
}
public void setMetadata(Post post, Metadata metadata) {
metadata.setPostIdRef(post.getId());
metadataRepository.save(metadata);
livePostsState.onNext(post.getId());
}
public Optional<Metadata> findMetadataByPostId(String postId) {
return metadataRepository.findByPostId(postId);
}
public void updateImageStatus(Status status, Long id) {
imageRepository.updateStatus(status, id);
liveImageUpdates.onNext(id);
}
public void updateImageCurrent(long current, Long id) {
imageRepository.updateCurrent(current, id);
liveImageUpdates.onNext(id);
}
public void updateImageTotal(long total, Long id) {
imageRepository.updateTotal(total, id);
liveImageUpdates.onNext(id);
}
public void updatePostStatus(Status status, Long id) {
postRepository.updateStatus(status, id);
livePostsState.onNext(id);
}
public void updatePostFolderName(String postFolderName, Long id) {
postRepository.updateFolderName(postFolderName, id);
livePostsState.onNext(id);
}
public void updatePostTitle(String title, Long id) {
postRepository.updateTitle(title, id);
livePostsState.onNext(id);
}
public void updatePostThanked(boolean thanked, Long id) {
postRepository.updateThanked(thanked, id);
livePostsState.onNext(id);
}
public void refreshPost(Long id) {
livePostsState.onNext(id);
}
}
@@ -5,8 +5,7 @@ import lombok.Getter;
@Getter
public class DownloadSpeed {
private final String type = "downSpeed";
private String speed;
private final String speed;
public DownloadSpeed(long bytes) {
speed = formatSI(bytes);
@@ -14,10 +14,11 @@ public class DownloadSpeedService {
private AtomicLong read = new AtomicLong(0);
@Getter
private long currentValue;
@Getter
private PublishProcessor<Long> readBytesPerSecond = PublishProcessor.create();
private final PublishProcessor<Long> readBytesPerSecond = PublishProcessor.create();
private boolean allowWrite = false;
@@ -7,15 +7,12 @@ import java.util.Objects;
@Getter
public class GlobalState {
private final String type = "globalState";
private long running;
private long queued;
private long remaining;
private long error;
private final long running;
private final long remaining;
private final long error;
GlobalState(long running, long queued, long remaining, long error) {
GlobalState(long running, long remaining, long error) {
this.running = running;
this.queued = queued;
this.remaining = remaining;
this.error = error;
}
@@ -25,11 +22,11 @@ public class GlobalState {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GlobalState that = (GlobalState) o;
return running == that.running && queued == that.queued && remaining == that.remaining && error == that.error;
return running == that.running && remaining == that.remaining && error == that.error;
}
@Override
public int hashCode() {
return Objects.hash(type, running, queued, remaining, error);
return Objects.hash(running, remaining, error);
}
}
@@ -6,46 +6,36 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.q.DownloadQ;
import tn.mnlr.vripper.q.ExecutionService;
import tn.mnlr.vripper.q.PendingQ;
@Service
@EnableScheduling
public class GlobalStateService {
private final DownloadQ downloadQ;
private final PendingQ pendingQ;
private final ExecutionService executionService;
private final AppStateExchange appStateExchange;
private final DataService dataService;
@Getter
private GlobalState currentState;
@Getter
private PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
private final PublishProcessor<GlobalState> liveGlobalState = PublishProcessor.create();
@Autowired
public GlobalStateService(DownloadQ downloadQ, ExecutionService executionService, AppStateExchange appStateExchange) {
this.downloadQ = downloadQ;
public GlobalStateService(PendingQ pendingQ, ExecutionService executionService, DataService dataService) {
this.pendingQ = pendingQ;
this.executionService = executionService;
this.appStateExchange = appStateExchange;
this.dataService = dataService;
}
@Scheduled(fixedDelay = 3000)
private void interval() {
GlobalState newGlobalState = new GlobalState(
executionService.runningCount(),
downloadQ.size(),
appStateExchange.getImages()
.values()
.stream()
.filter(e -> e.getTotal() == 0 || e.getTotal() != e.getCurrent().get())
.count(),
appStateExchange.getImages()
.values()
.stream()
.filter(e -> e.getStatus().equals(Image.Status.ERROR))
.count());
pendingQ.size(),
dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
liveGlobalState.onNext(currentState);
@@ -0,0 +1,27 @@
package tn.mnlr.vripper.services;
import org.springframework.stereotype.Service;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
@Service
public class MutexService {
private final Map<String, ReentrantLock> postLock = new ConcurrentHashMap<>();
public synchronized void createPostLock(String postId) {
if (!postLock.containsKey(postId)) {
postLock.put(postId, new ReentrantLock());
}
}
public void removePostLock(String postId) {
postLock.remove(postId);
}
public ReentrantLock getPostLock(String postId) {
return postLock.get(postId);
}
}
@@ -1,37 +1,39 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.jpa.domain.Post;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
@Service
@Slf4j
public class PathService {
private static final Logger logger = LoggerFactory.getLogger(PathService.class);
private final AppSettingsService appSettingsService;
private final DataService dataService;
private final MutexService mutexService;
private final CommonExecutor commonExecutor;
private final int MAX_DELETE_ATTEMPTS = 180;
@Getter
private final Set<String> renaming = Collections.synchronizedSet(new HashSet<>());
@Autowired
public PathService(AppSettingsService appSettingsService, CommonExecutor commonExecutor) {
public PathService(AppSettingsService appSettingsService, DataService dataService, MutexService mutexService, CommonExecutor commonExecutor) {
this.appSettingsService = appSettingsService;
this.dataService = dataService;
this.mutexService = mutexService;
this.commonExecutor = commonExecutor;
}
@@ -45,48 +47,68 @@ public class PathService {
return new File(sourceFolder, title);
}
public synchronized final void createDefaultPostFolder(Post post) {
public final void createDefaultPostFolder(Post post) {
File sourceFolder = _getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(post.getTitle()));
File destFolder = makeDirs(sourceFolder);
post.setPostFolderName(destFolder.getName());
dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
}
public synchronized final void rename(Post post, String altName) {
post.setTitle(altName);
if (post.getPostFolderName() == null) {
return;
}
File newDestFolder = makeDirs(_getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(altName)));
File currentDesFolder = getDownloadDestinationFolder(post);
post.setPostFolderName(newDestFolder.getName());
List<File> files = Optional.ofNullable(currentDesFolder.listFiles()).stream().flatMap(Arrays::stream).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
for (File f : files) {
try {
Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
logger.error(String.format("Failed to move files from %s to %s", currentDesFolder.toString(), newDestFolder.toString()), e);
return;
}
}
public final void rename(@NonNull String postId, @NonNull String altName) {
renaming.add(postId);
Post post = dataService.findPostByPostId(postId).orElseThrow();
dataService.refreshPost(post.getId());
commonExecutor.getGeneralExecutor().submit(() -> {
int attemptCount = 0;
while (Objects.requireNonNull(currentDesFolder.listFiles()).length != 0 && attemptCount < MAX_DELETE_ATTEMPTS) {
try {
attemptCount++;
Thread.sleep(1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
ReentrantLock postLock = null;
try {
postLock = mutexService.getPostLock(postId);
if (postLock != null) {
postLock.lock();
}
if (altName.equals(post.getTitle())) {
return;
}
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
if (post.getPostFolderName() == null) {
return;
}
File newDestFolder = makeDirs(_getDownloadDestinationFolder(post.getForum(), post.getThreadTitle(), sanitize(altName)));
File currentDesFolder = getDownloadDestinationFolder(post);
post.setPostFolderName(newDestFolder.getName());
dataService.updatePostFolderName(post.getPostFolderName(), post.getId());
List<File> files = Arrays.stream(Objects.requireNonNull(currentDesFolder.listFiles())).filter(e -> !e.getName().endsWith(".tmp")).collect(Collectors.toList());
for (File f : files) {
try {
Files.move(f.toPath(), Paths.get(newDestFolder.toString(), f.toPath().getFileName().toString()), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
log.error(String.format("Failed to move files from %s to %s", currentDesFolder.toString(), newDestFolder.toString()), e);
return;
}
}
for (File file : Objects.requireNonNull(currentDesFolder.listFiles())) {
if (!file.delete()) {
log.warn(String.format("Failed to remove %s", file.toString()));
}
}
if (!currentDesFolder.delete()) {
log.warn(String.format("Failed to remove %s", currentDesFolder.toString()));
}
} finally {
renaming.remove(postId);
dataService.refreshPost(post.getId());
if (postLock != null) {
postLock.unlock();
}
}
if (!currentDesFolder.delete()) {
logger.warn(String.format("Failed to remove %s", currentDesFolder.toString()));
}
});
}
private synchronized File makeDirs(@NonNull final File sourceFolder) {
private File makeDirs(@NonNull final File sourceFolder) {
int counter = 1;
File folder = sourceFolder;
@@ -103,7 +125,7 @@ public class PathService {
private String sanitize(final String folderName) {
String sanitizedFolderName = folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
logger.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
log.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
return sanitizedFolderName;
}
@@ -1,146 +0,0 @@
package tn.mnlr.vripper.services;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.PublishProcessor;
import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.EventListenerBean;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.entities.mixin.persistance.ImagePersistanceMixin;
import tn.mnlr.vripper.entities.mixin.persistance.PostPersistanceMixin;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@Service
public class PersistenceService {
private static final Logger logger = LoggerFactory.getLogger(PersistenceService.class);
private final AppStateExchange appStateExchange;
@Value("${base.dir}")
private String baseDir;
@Getter
private String dataPath;
private ObjectMapper om;
private Disposable subscription;
@Getter
private PublishProcessor<Map<String, Post>> processor = PublishProcessor.create();
@Autowired
private PersistenceService(AppStateExchange appStateExchange) {
this.appStateExchange = appStateExchange;
om = new ObjectMapper();
om.addMixIn(Image.class, ImagePersistanceMixin.class);
om.addMixIn(Post.class, PostPersistanceMixin.class);
subscription = processor
.onBackpressureBuffer()
.buffer(1, TimeUnit.SECONDS)
.filter(e -> !e.isEmpty())
.doOnNext(e -> this.persist(e.get(0)))
.subscribe();
}
@PostConstruct
public void init() {
dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
File dataFile = new File(dataPath);
if (!dataFile.exists()) {
try {
if (dataFile.getParentFile().mkdirs()) {
logger.debug(String.format("%s is created", dataFile.getParentFile().toString()));
}
if (!dataFile.getParentFile().isDirectory() || !dataFile.getParentFile().canWrite()) {
logger.error(String.format("Unable to write in %s", dataFile.getParent()));
SpringContext.close();
}
if (dataFile.createNewFile()) {
logger.debug("Data file successfully created");
try (FileWriter fw = new FileWriter(dataFile)) {
fw.write("{}");
}
} else {
logger.warn("Data file already exists");
}
} catch (IOException e) {
logger.error("Unable to create data file", e);
SpringContext.close();
}
}
}
@PreDestroy
public void preDestroy() {
this.subscription.dispose();
logger.info(String.format("Destroying %s", PersistenceService.class.getSimpleName()));
logger.info("Persisting data before destroying");
if (EventListenerBean.isInit()) {
this.persist(appStateExchange.getPosts());
}
}
private void persist(Map<String, Post> currentPosts) {
try (PrintWriter out = new PrintWriter(dataPath, StandardCharsets.UTF_8)) {
out.print(om.writeValueAsString(currentPosts));
} catch (IOException e) {
logger.error("Failed to persist app state", e);
}
}
private Map<String, Post> read(String content) {
try {
return om.readValue(content, om.getTypeFactory().constructMapType(HashMap.class, String.class, Post.class));
} catch (IOException e) {
logger.error("Failed to read app state", e);
long timestamp = new Date().getTime();
logger.warn(String.format("trying to rename old data file from %s to %s", dataPath, dataPath + "." + timestamp + ".old"));
try {
Files.move(new File(dataPath).toPath(), new File(dataPath + "." + timestamp + ".old").toPath());
} catch (IOException ex) {
logger.error(String.format("Failed to rename %s to %s", dataPath, dataPath + ".old"));
SpringContext.close();
}
}
return new HashMap<>();
}
public void restore() {
String jsonContent = null;
try {
jsonContent = String.join("", Files.readAllLines(Paths.get(dataPath), StandardCharsets.UTF_8));
} catch (Exception e) {
logger.error("data file cannot be read, previous state cannot be restored", e);
SpringContext.close();
}
appStateExchange.restore(read(jsonContent));
}
}
@@ -1,60 +0,0 @@
package tn.mnlr.vripper.services;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.q.DownloadQ;
@Service
public class PostParser {
private static final Logger logger = LoggerFactory.getLogger(PostParser.class);
static final String VR_API = "https://vipergirls.to/vr.php";
private final AppStateExchange appStateExchange;
private final AppSettingsService appSettingsService;
private final DownloadQ downloadQ;
@Autowired
public PostParser(AppStateExchange appStateExchange, AppSettingsService appSettingsService, DownloadQ downloadQ) {
this.appStateExchange = appStateExchange;
this.appSettingsService = appSettingsService;
this.downloadQ = downloadQ;
}
public synchronized void addPost(String postId, String threadId) throws PostParseException {
if (appStateExchange.getPosts().containsKey(postId)) {
logger.warn(String.format("skipping %s, already loaded", postId));
return;
}
VRPostParser vrPostParser = new VRPostParser(threadId, postId);
Post post = vrPostParser.parse();
if (appSettingsService.getSettings().getAutoStart()) {
logger.debug("Auto start downloads option is enabled");
logger.debug(String.format("Starting to enqueue %d jobs for %s", post.getImages().size(), post.getUrl()));
post.setStatus(Post.Status.PENDING);
try {
downloadQ.enqueue(post);
} catch (InterruptedException e) {
logger.warn("Interruption was caught");
Thread.currentThread().interrupt();
return;
}
logger.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Post.Status.STOPPED);
logger.debug("Auto start downloads option is disabled");
}
}
public VRThreadParser createVRThreadParser(QueuedVGLink queuedVGLink) {
return new VRThreadParser(queuedVGLink);
}
}
@@ -1,64 +0,0 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.ToString;
import tn.mnlr.vripper.SpringContext;
import java.util.Objects;
@ToString
@Getter
public class QueuedVGLink {
private final String type = "grabQueue";
private final AppStateService appStateService;
private final String link;
private final String threadId;
private final String postId;
private int count = 0;
private boolean loading = true;
private boolean removed = false;
public QueuedVGLink(String link, String threadId, String postId) {
this.appStateService = SpringContext.getBean(AppStateService.class);
this.link = link;
this.threadId = threadId;
this.postId = postId;
}
public void done() {
this.loading = false;
this.appStateService.queueLinkUpdated(this);
}
public void increment() {
this.count++;
this.appStateService.queueLinkUpdated(this);
}
public void setCount(int count) {
this.count = count;
}
public void remove() {
this.removed = true;
appStateService.queueLinkUpdated(this);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
QueuedVGLink that = (QueuedVGLink) o;
return Objects.equals(threadId, that.threadId);
}
@Override
public int hashCode() {
return Objects.hash(threadId);
}
}
@@ -1,173 +0,0 @@
package tn.mnlr.vripper.services;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.Weigher;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.imgscalr.Scalr;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.FileSystemUtils;
import tn.mnlr.vripper.entities.Post;
import javax.annotation.PostConstruct;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Duration;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class ThumbnailGenerator {
@Getter
private LoadingCache<CacheKey, byte[]> thumbnails;
@Value("${base.dir}")
private String baseDir;
private final PathService pathService;
private final AppStateExchange appStateExchange;
@Getter
private File cacheFolder;
CacheLoader<CacheKey, byte[]> loader = new CacheLoader<>() {
@Override
public byte[] load(CacheKey key) throws Exception {
Post post = appStateExchange.getPost(key.getPostId());
File destinationFolder = pathService.getDownloadDestinationFolder(post);
if (!destinationFolder.exists() || !destinationFolder.isDirectory()) {
return null;
}
return Files.readAllBytes(generateThumbnail(new File(destinationFolder, key.getImgName()), key.getPostId()).toPath());
}
};
@Autowired
public ThumbnailGenerator(PathService pathService, AppStateExchange appStateExchange) {
this.pathService = pathService;
this.appStateExchange = appStateExchange;
}
@PostConstruct
private void init() throws Exception {
cacheFolder = new File(baseDir + File.separator + ".vripper" + File.separator + "cache");
cacheFolder.mkdirs();
if (!cacheFolder.exists()) {
throw new Exception(String.format("%s could not be created", cacheFolder.toString()));
}
thumbnails = CacheBuilder.newBuilder()
.expireAfterAccess(Duration.ofMinutes(30))
.weigher((Weigher<CacheKey, byte[]>) (k, v) -> v.length)
.maximumWeight(104_857_600)
.build(loader);
}
public void clearCache() {
thumbnails.invalidateAll();
for (File file : Optional.ofNullable(cacheFolder.listFiles()).orElse(new File[]{})) {
FileSystemUtils.deleteRecursively(file);
}
}
public long cacheSize() {
final AtomicLong size = new AtomicLong(0);
try {
Files.walkFileTree(cacheFolder.toPath(), new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
size.addAndGet(attrs.size());
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
System.out.println("skipped: " + file + " (" + exc + ")");
// Skip folders that can't be traversed
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
if (exc != null)
System.out.println("had trouble traversing: " + dir + " (" + exc + ")");
else
size.addAndGet(dir.toFile().length());
// Ignore errors traversing a folder
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
throw new AssertionError("walkFileTree will not throw IOException if the FileVisitor does not");
}
return size.get();
}
private File generateThumbnail(File inputFile, String postId) throws Exception {
if (!inputFile.exists()) {
throw new Exception(String.format("Input file %s does not exist", inputFile.toString()));
}
File postsCacheFolder = new File(cacheFolder, postId);
postsCacheFolder.mkdirs();
if (!postsCacheFolder.exists()) {
throw new Exception(String.format("%s could not be created", postsCacheFolder.toString()));
}
File thumbFile = new File(postsCacheFolder, inputFile.getName());
if (thumbFile.exists()) {
return thumbFile;
}
BufferedImage image = ImageIO.read(inputFile);
BufferedImage resize = Scalr.resize(image, 350, 350);
ImageIO.write(resize, "jpg", thumbFile);
image.flush();
resize.flush();
return thumbFile;
}
@Getter
@Setter
@NoArgsConstructor
public static class CacheKey {
private String postId;
private String imgName;
public CacheKey(String postId, String imgName) {
this.postId = postId;
this.imgName = imgName;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CacheKey cacheKey = (CacheKey) o;
return Objects.equals(postId, cacheKey.postId) &&
Objects.equals(imgName, cacheKey.imgName);
}
@Override
public int hashCode() {
return Objects.hash(postId, imgName);
}
}
}
@@ -5,37 +5,39 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.post.CachedPost;
import tn.mnlr.vripper.services.post.PostService;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@Service
@Slf4j
public class VGHandler {
private static final Logger logger = LoggerFactory.getLogger(VGHandler.class);
private final AppStateService appStateService;
private final PostParser postParser;
private final DataService dataService;
private final PostService postService;
private final CommonExecutor commonExecutor;
@Getter
private final LoadingCache<QueuedVGLink, List<VRPostState>> cache;
private final LoadingCache<Queued, List<CachedPost>> cache;
@Autowired
public VGHandler(AppStateService appStateService, PostParser postParser, CommonExecutor commonExecutor) {
this.appStateService = appStateService;
this.postParser = postParser;
public VGHandler(DataService dataService, PostService postService, CommonExecutor commonExecutor) {
this.dataService = dataService;
this.postService = postService;
this.commonExecutor = commonExecutor;
CacheLoader<QueuedVGLink, List<VRPostState>> loader = new CacheLoader<>() {
CacheLoader<Queued, List<CachedPost>> loader = new CacheLoader<>() {
@Override
public List<VRPostState> load(@NonNull QueuedVGLink queuedVGLink) throws Exception {
VRThreadParser vrThreadParser = postParser.createVRThreadParser(queuedVGLink);
public List<CachedPost> load(@NonNull Queued queuedVGLink) throws Exception {
VRThreadParser vrThreadParser = new VRThreadParser(queuedVGLink);
return vrThreadParser.parse();
}
};
@@ -45,31 +47,40 @@ public class VGHandler {
.build(loader);
}
public void handle(List<QueuedVGLink> queuedVGLinks) throws Exception {
for (QueuedVGLink queuedVGLink : queuedVGLinks) {
if (queuedVGLink.getPostId() != null) {
postParser.addPost(queuedVGLink.getPostId(), queuedVGLink.getThreadId());
public void handle(List<Queued> queuedList) throws Exception {
for (Queued queued : queuedList) {
if (queued.getPostId() != null) {
postService.addPost(queued.getPostId(), queued.getThreadId());
} else {
Callable<Void> cl = () -> {
List<VRPostState> vrPostStates = cache.get(queuedVGLink);
queuedVGLink.setCount(vrPostStates.size());
queuedVGLink.done();
logger.debug(String.format("%d found for %s", vrPostStates.size(), queuedVGLink.getLink()));
if (vrPostStates.size() == 1) {
queuedVGLink.remove();
postParser.addPost(vrPostStates.get(0).getPostId(), vrPostStates.get(0).getThreadId());
logger.debug(String.format("threadId %s, postId %s is added automatically for download", queuedVGLink.getThreadId(), queuedVGLink.getPostId()));
Runnable runnable = () -> {
List<CachedPost> cachedPosts;
try {
cachedPosts = cache.get(queued);
} catch (ExecutionException e) {
log.error(String.format("Failed to add post with thread id %s, postId %s", queued.getThreadId(), queued.getPostId()), e);
return;
}
queued.setTotal(cachedPosts.size());
queued.done();
log.debug(String.format("%d found for %s", cachedPosts.size(), queued.getLink()));
if (cachedPosts.size() == 1) {
try {
postService.addPost(cachedPosts.get(0).getPostId(), cachedPosts.get(0).getThreadId());
} catch (PostParseException e) {
log.error(String.format("Failed to add post with postId %s", cachedPosts.get(0).getPostId()), e);
return;
}
log.debug(String.format("threadId %s, postId %s is added automatically for download", queued.getThreadId(), queued.getPostId()));
} else {
appStateService.newQueueLink(queuedVGLink);
dataService.newQueueLink(queued);
}
return null;
};
commonExecutor.getGeneralExecutor().submit(cl);
commonExecutor.getGeneralExecutor().submit(runnable);
}
}
}
public void remove(String threadId) {
appStateService.removeQueueLink(threadId);
dataService.removeQueueLink(threadId);
}
}
@@ -1,263 +0,0 @@
package tn.mnlr.vripper.services;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.host.Host;
import javax.xml.parsers.SAXParserFactory;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import static tn.mnlr.vripper.services.PostParser.VR_API;
class VRPostParser {
private static final Logger logger = LoggerFactory.getLogger(VRPostParser.class);
private static SAXParserFactory factory = SAXParserFactory.newInstance();
private static List<String> dictionary = Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
private final String threadId;
private final String postId;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
private final HtmlProcessorService htmlProcessorService;
private final XpathService xpathService;
VRPostParser(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
this.cm = SpringContext.getBean(ConnectionManager.class);
this.vipergirlsAuthService = SpringContext.getBean(VipergirlsAuthService.class);
this.htmlProcessorService = SpringContext.getBean(HtmlProcessorService.class);
this.xpathService = SpringContext.getBean(XpathService.class);
}
public Post parse() throws PostParseException {
logger.debug(String.format("Parsing post %s", postId));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(VR_API);
uriBuilder.setParameter("p", postId);
httpGet = cm.buildHttpGet(uriBuilder.build());
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
AtomicReference<Throwable> thr = new AtomicReference<>();
HashMap<String, Object> metadata = new HashMap<>();
String postUrl = String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", threadId, postId, postId);
getPostExtraMetadata(postId, metadata, postUrl, thr);
if (thr.get() != null) {
logger.warn(String.format("Failed to get metadata for thread %s, post %s", threadId, postId), thr.get());
thr.set(null);
}
VRApiPostHandler handler = new VRApiPostHandler(threadId, postId, postUrl, metadata);
logger.debug(String.format("Requesting %s", httpGet));
Optional<Post> post = getPost(httpGet, handler, thr);
if (thr.get() != null || post.isEmpty()) {
logger.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
throw new PostParseException(thr.get());
}
return post.get();
}
private Optional<Post> getPost(HttpGet httpGet, VRApiPostHandler handler, AtomicReference<Throwable> thr) {
return Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
factory.newSAXParser().parse(response.getEntity().getContent(), handler);
EntityUtils.consumeQuietly(response.getEntity());
return handler.getParsedPost();
}
});
}
private void getPostExtraMetadata(String postId, HashMap<String, Object> metadata, String url, AtomicReference<Throwable> thr) {
HttpGet httpGet = cm.buildHttpGet(url);
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
try {
Document document = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
Node postNode = xpathService.getAsNode(document, String.format("//li[@id='post_%s']/div[contains(@class, 'postdetails')]", postId));
String postedBy = xpathService.getAsNode(postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font").getTextContent().trim();
metadata.put(Post.METADATA.POSTED_BY.name(), postedBy);
Node node = xpathService.getAsNode(document, String.format("//div[@id='post_message_%s']", postId));
metadata.put(Post.METADATA.RESOLVED_NAME.name(), findTitleInContent(node));
return null;
} catch (Exception e) {
throw new PostParseException(String.format("Failed to parse thread %s, post %s", threadId, postId), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
}
private List<String> findTitleInContent(Node node) {
List<String> altTitle = new ArrayList<>();
findTitle(node, altTitle);
return altTitle;
}
private boolean findTitle(Node node, List<String> altTitle) {
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
return false;
}
if (node.getNodeType() == Node.ELEMENT_NODE) {
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
Node item = node.getChildNodes().item(i);
findTitle(item, altTitle);
}
} else if (node.getNodeType() == Node.TEXT_NODE) {
String text = node.getTextContent().trim();
if (!text.isEmpty() && dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
altTitle.add(text);
}
}
return true;
}
}
class VRApiPostHandler extends DefaultHandler {
private static final Logger logger = LoggerFactory.getLogger(VRApiPostHandler.class);
private final Collection<Host> supportedHosts;
private final String threadId;
private final String postId;
private final HashMap<String, Object> metadata;
private final String postUrl;
private List<Image> images = new ArrayList<>();
private List<String> previews = new ArrayList<>();
private String threadTitle;
private String postTitle;
private String forum;
private int previewCounter = 0;
private int index = 0;
private int imageCount;
private Post parsedPost;
VRApiPostHandler(String threadId, String postId, String postUrl, HashMap<String, Object> metadata) {
this.threadId = threadId;
this.postId = postId;
this.postUrl = postUrl;
this.metadata = metadata;
this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
}
public Optional<Post> getParsedPost() {
return Optional.ofNullable(parsedPost);
}
@Override
public void startDocument() {
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "forum":
forum = attributes.getValue("title").trim();
break;
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
break;
case "image":
index++;
if (previewCounter++ < 4) {
Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
}
String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
if (mainUrl != null) {
Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
if (foundHost != null) {
logger.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
try {
images.add(new Image(mainUrl, postId, postTitle, foundHost, index));
} catch (PostParseException e) {
logger.error(String.format("Error occurred while parsing postId %s", postId));
}
} else {
logger.warn(String.format("unsupported host for %s, skipping", mainUrl));
}
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equals(qName.toLowerCase())) {
if (imageCount != 0) {
metadata.put(Post.METADATA.PREVIEWS.name(), previews);
AtomicReference<Throwable> thr = new AtomicReference<>();
if (thr.get() != null) {
logger.error(String.format("Failed to get extra metadata for %s", postUrl), thr.get());
}
try {
parsedPost = new Post(
postTitle,
postUrl,
images,
metadata,
postId,
threadId,
threadTitle,
forum);
} catch (PostParseException e) {
logger.error(String.format("Error occurred while parsing postId %s", postId));
}
}
index = 0;
previewCounter = 0;
previews = new ArrayList<>();
images = new ArrayList<>();
}
}
}
@@ -1,29 +0,0 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import java.util.List;
@Getter
public class VRPostState {
private final String type = "postParse";
private final String threadId;
private String postId;
private int number;
private String title;
private int imageCount;
private String url;
private List<String> previews;
private String hosts;
VRPostState(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
this.threadId = threadId;
this.postId = postId;
this.number = number;
this.title = title;
this.imageCount = imageCount;
this.previews = previews;
this.url = url;
this.hosts = hosts;
}
}
@@ -1,14 +1,13 @@
package tn.mnlr.vripper.services;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
@@ -16,6 +15,8 @@ import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.post.CachedPost;
import javax.xml.parsers.SAXParserFactory;
import java.io.BufferedInputStream;
@@ -25,39 +26,38 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static tn.mnlr.vripper.services.PostParser.VR_API;
@Slf4j
public class VRThreadParser {
private static final Logger logger = LoggerFactory.getLogger(VRThreadParser.class);
private static final String VR_API = "https://vipergirls.to/vr.php";
private static SAXParserFactory factory = SAXParserFactory.newInstance();
private final QueuedVGLink queuedVGLink;
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final Queued queued;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
VRThreadParser(QueuedVGLink queuedVGLink) {
this.queuedVGLink = queuedVGLink;
VRThreadParser(Queued queued) {
this.queued = queued;
this.cm = SpringContext.getBean(ConnectionManager.class);
this.vipergirlsAuthService = SpringContext.getBean(VipergirlsAuthService.class);
}
public List<VRPostState> parse() throws PostParseException {
public List<CachedPost> parse() throws PostParseException {
logger.debug(String.format("Parsing thread %s", queuedVGLink));
log.debug(String.format("Parsing thread %s", queued));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(VR_API);
uriBuilder.setParameter("t", queuedVGLink.getThreadId());
uriBuilder.setParameter("t", queued.getThreadId());
httpGet = cm.buildHttpGet(uriBuilder.build());
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
VRThreadHandler handler = new VRThreadHandler(queuedVGLink);
VRThreadHandler handler = new VRThreadHandler(queued);
AtomicReference<Throwable> thr = new AtomicReference<>();
logger.debug(String.format("Requesting %s", httpGet));
List<VRPostState> posts = Failsafe.with(VripperApplication.retryPolicy)
log.debug(String.format("Requesting %s", httpGet));
List<CachedPost> posts = Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
@@ -70,14 +70,14 @@ public class VRThreadParser {
factory.newSAXParser().parse(new BufferedInputStream(response.getEntity().getContent()), handler);
return handler.getPosts();
} catch (Exception e) {
throw new PostParseException(String.format("Failed to parse thread %s", queuedVGLink), e);
throw new PostParseException(String.format("Failed to parse thread %s", queued), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
if (thr.get() != null) {
logger.error(String.format("parsing failed for thread %s", queuedVGLink), thr.get());
log.error(String.format("parsing failed for thread %s", queued), thr.get());
throw new PostParseException(thr.get());
}
return posts;
@@ -86,7 +86,7 @@ public class VRThreadParser {
class VRThreadHandler extends DefaultHandler {
private final QueuedVGLink queuedVGLink;
private final Queued queued;
private final Collection<Host> supportedHosts;
private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
private List<String> previews = new ArrayList<>();
@@ -98,10 +98,10 @@ class VRThreadHandler extends DefaultHandler {
private int previewCounter = 0;
@Getter
private List<VRPostState> posts = new ArrayList<>();
private List<CachedPost> posts = new ArrayList<>();
VRThreadHandler(QueuedVGLink queuedVGLink) {
this.queuedVGLink = queuedVGLink;
VRThreadHandler(Queued queued) {
this.queued = queued;
this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
}
@@ -138,17 +138,17 @@ class VRThreadHandler extends DefaultHandler {
public void endElement(String uri, String localName, String qName) {
if ("post".equals(qName.toLowerCase())) {
if (imageCount != 0) {
posts.add(new VRPostState(
queuedVGLink.getThreadId(),
posts.add(new CachedPost(
queued.getThreadId(),
postId,
postCounter,
postTitle,
imageCount,
String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", queuedVGLink, postId, postId),
String.format("https://vipergirls.to/threads/?p=%s&viewfull=1#post%s", postId, postId),
previews,
hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
));
queuedVGLink.increment();
queued.increment();
}
previewCounter = 0;
previews = new ArrayList<>();
@@ -158,6 +158,6 @@ class VRThreadHandler extends DefaultHandler {
@Override
public void endDocument() {
queuedVGLink.done();
queued.done();
}
}
@@ -2,10 +2,10 @@ package tn.mnlr.vripper.services;
import io.reactivex.processors.PublishProcessor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.cookie.Cookie;
@@ -13,49 +13,42 @@ import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.jpa.domain.Post;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Service
@Slf4j
public class VipergirlsAuthService {
private static final Logger logger = LoggerFactory.getLogger(VipergirlsAuthService.class);
private final ConnectionManager cm;
private final HtmlProcessorService htmlProcessorService;
private final XpathService xpathService;
private final AppSettingsService appSettingsService;
private final CommonExecutor commonExecutor;
private final DataService dataService;
@Getter
private HttpClientContext context = HttpClientContext.create();
private final HttpClientContext context = HttpClientContext.create();
@Getter
private boolean authenticated = false;
@Getter
private String loggedUser;
private String loggedUser = "";
@Getter
private PublishProcessor<String> loggedInUser = PublishProcessor.create();
private final PublishProcessor<String> loggedInUser = PublishProcessor.create();
@Autowired
public VipergirlsAuthService(ConnectionManager cm, HtmlProcessorService htmlProcessorService, XpathService xpathService, AppSettingsService appSettingsService, CommonExecutor commonExecutor) {
public VipergirlsAuthService(ConnectionManager cm, AppSettingsService appSettingsService, CommonExecutor commonExecutor, DataService dataService) {
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
this.xpathService = xpathService;
this.appSettingsService = appSettingsService;
this.commonExecutor = commonExecutor;
this.dataService = dataService;
}
@PostConstruct
@@ -64,17 +57,17 @@ public class VipergirlsAuthService {
try {
authenticate();
} catch (VripperException e) {
logger.error("Cannot authenticate user with ViperGirls", e);
log.error("Cannot authenticate user with ViperGirls", e);
}
}
public void authenticate() throws VripperException {
logger.info("Authenticating using ViperGirls credentials");
log.info("Authenticating using ViperGirls credentials");
authenticated = false;
if (!appSettingsService.getSettings().getVLogin()) {
logger.debug("Authentication option is disabled");
log.debug("Authentication option is disabled");
context.getCookieStore().clear();
loggedUser = "";
loggedInUser.onNext(loggedUser);
@@ -85,7 +78,7 @@ public class VipergirlsAuthService {
String password = appSettingsService.getSettings().getVPassword();
if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
logger.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
context.getCookieStore().clear();
loggedUser = "";
loggedInUser.onNext(loggedUser);
@@ -95,14 +88,10 @@ public class VipergirlsAuthService {
HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login");
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("vb_login_username", username));
params.add(new BasicNameValuePair("vb_login_password", ""));
params.add(new BasicNameValuePair("vb_login_password_hint", "Password"));
params.add(new BasicNameValuePair("cookieuser", "1"));
params.add(new BasicNameValuePair("securitytoken", "guest"));
params.add(new BasicNameValuePair("do", "login"));
params.add(new BasicNameValuePair("vb_login_md5password", password));
params.add(new BasicNameValuePair("vb_login_md5password_utf", password));
try {
postAuth.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
@@ -123,7 +112,7 @@ public class VipergirlsAuthService {
throw new VripperException(String.format("Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
}
String responseBody = EntityUtils.toString(response.getEntity());
logger.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
log.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
EntityUtils.consumeQuietly(response.getEntity());
if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) {
throw new VripperException("Failed to authenticate user with ViperRipper");
@@ -140,79 +129,44 @@ public class VipergirlsAuthService {
}
authenticated = true;
loggedUser = username;
logger.info(String.format("Authenticated: %s", username));
log.info(String.format("Authenticated: %s", username));
loggedInUser.onNext(loggedUser);
}
public void leaveThanks(Post post) {
if (!appSettingsService.getSettings().getVLogin()) {
logger.debug("Authentication with ViperGirls option is disabled");
log.debug("Authentication with ViperGirls option is disabled");
return;
}
if (!appSettingsService.getSettings().getVThanks()) {
logger.debug("Leave thanks option is disabled");
log.debug("Leave thanks option is disabled");
return;
}
if (!authenticated) {
logger.error("You are not authenticated");
log.error("You are not authenticated");
return;
}
if (post.getMetadata().get(Post.METADATA.THANKED.name()) != null && (boolean) post.getMetadata().get(Post.METADATA.THANKED.name())) {
logger.debug("Already left a thanks");
if (post.isThanked()) {
log.debug("Already left a thanks");
return;
}
commonExecutor.getGeneralExecutor().submit(() -> {
try {
postThanks(post, getSecurityToken(post.getUrl()));
postThanks(post);
} catch (Exception e) {
logger.error(String.format("Failed to leave a thanks for url %s, post id %s", post.getUrl(), post.getPostId()), e);
log.error(String.format("Failed to leave a thanks for url %s, post id %s", post.getUrl(), post.getPostId()), e);
}
});
}
private String getSecurityToken(String url) throws VripperException {
String securityToken;
HttpGet httpGet = cm.buildHttpGet(url);
httpGet.addHeader("Referer", "https://vipergirls.to/");
httpGet.addHeader("Host", "vipergirls.to");
CloseableHttpClient client = cm.getClient().build();
try (CloseableHttpResponse response = client.execute(httpGet, context)) {
String postPage = EntityUtils.toString(response.getEntity());
Document document = htmlProcessorService.clean(postPage);
String thanksUrl = xpathService
.getAsNode(document, "//li[contains(@id,'post_')][not(contains(@id,'post_thank'))]//a[@class='post_thanks_button']")
.getAttributes()
.getNamedItem("href")
.getTextContent()
.trim();
securityToken = Arrays.stream(thanksUrl.split("&amp;"))
.filter(v -> v.startsWith("securitytoken"))
.findAny()
.orElse("")
.replace("securitytoken=", "");
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
throw new VripperException(e);
}
return securityToken;
}
private void postThanks(Post post, String securityKey) throws VripperException {
private void postThanks(Post post) throws VripperException {
HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php");
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("do", "post_thanks_add"));
params.add(new BasicNameValuePair("using_ajax", "1"));
params.add(new BasicNameValuePair("p", post.getPostId()));
params.add(new BasicNameValuePair("securitytoken", securityKey));
params.add(new BasicNameValuePair("securitytoken", post.getSecurityToken()));
try {
postThanks.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
@@ -225,10 +179,17 @@ public class VipergirlsAuthService {
CloseableHttpClient client = cm.getClient().build();
try (CloseableHttpResponse response = client.execute(postThanks, context)) {
post.getMetadata().put(Post.METADATA.THANKED.name(), true);
if (response.getStatusLine().getStatusCode() / 100 == 2) {
post.setThanked(true);
dataService.updatePostThanked(post.isThanked(), post.getId());
}
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
throw new VripperException(e);
}
if (!post.isThanked()) {
throw new VripperException("Failed to leave");
}
}
}
@@ -0,0 +1,29 @@
package tn.mnlr.vripper.services.post;
import lombok.Getter;
import java.util.List;
@Getter
public class CachedPost {
private final String type = "postParse";
private final String threadId;
private final String postId;
private final int number;
private final String title;
private final int imageCount;
private final String url;
private final List<String> previews;
private final String hosts;
public CachedPost(String threadId, String postId, int number, String title, int imageCount, String url, List<String> previews, String hosts) {
this.threadId = threadId;
this.postId = postId;
this.number = number;
this.title = title;
this.imageCount = imageCount;
this.previews = previews;
this.url = url;
this.hosts = hosts;
}
}
@@ -0,0 +1,169 @@
package tn.mnlr.vripper.services.post;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import tn.mnlr.vripper.services.XpathService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@Slf4j
@Service
public class MetadataCache {
@Getter
static class Key {
private final String postId;
private final String threadId;
private final String url;
Key(String postId, String threadId, String url) {
this.postId = postId;
this.threadId = threadId;
this.url = url;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Key key = (Key) o;
return Objects.equals(postId, key.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
private static final List<String> dictionary = Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
private final LoadingCache<Key, Metadata> cache;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
private final HtmlProcessorService htmlProcessorService;
private final XpathService xpathService;
@Autowired
public MetadataCache(ConnectionManager cm, VipergirlsAuthService vipergirlsAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService) {
this.cm = cm;
this.vipergirlsAuthService = vipergirlsAuthService;
this.htmlProcessorService = htmlProcessorService;
this.xpathService = xpathService;
CacheLoader<Key, Metadata> loader = new CacheLoader<>() {
@Override
public Metadata load(@NonNull Key key) {
return fetchMetadata(key);
}
};
cache = CacheBuilder.newBuilder()
.expireAfterWrite(30, TimeUnit.MINUTES)
.build(loader);
}
public Metadata get(Post post) throws ExecutionException {
Metadata metadata = new Metadata();
Metadata cachedMetadata = cache.get(new Key(post.getPostId(), post.getThreadId(), post.getUrl()));
metadata.setPostedBy(cachedMetadata.getPostedBy());
metadata.setResolvedNames(List.copyOf(cachedMetadata.getResolvedNames()));
return metadata;
}
private Metadata fetchMetadata(Key key) {
HttpGet httpGet = cm.buildHttpGet(key.getUrl());
Metadata metadata = new Metadata();
Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> {
if (e.getFailure() instanceof InterruptedException || e.getFailure().getCause() instanceof InterruptedException) {
log.debug("Fetching interrupted");
return;
}
log.error(String.format("Error occurred when getting post metadata, postId %s", key.getPostId()), e.getFailure());
})
.run(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
try {
if (Thread.interrupted()) {
return;
}
Document document = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
Node postNode = xpathService.getAsNode(document, String.format("//li[@id='post_%s']/div[contains(@class, 'postdetails')]", key.getPostId()));
String postedBy = xpathService.getAsNode(postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font").getTextContent().trim();
metadata.setPostedBy(postedBy);
Node node = xpathService.getAsNode(document, String.format("//div[@id='post_message_%s']", key.getPostId()));
metadata.setResolvedNames(findTitleInContent(node));
} catch (Exception e) {
throw new PostParseException(String.format("Failed to parse thread %s, post %s", key.getThreadId(), key.getPostId()), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
return metadata;
}
private List<String> findTitleInContent(Node node) {
List<String> altTitle = new ArrayList<>();
findTitle(node, altTitle, new AtomicBoolean(true));
return altTitle.stream().distinct().collect(Collectors.toList());
}
private void findTitle(Node node, List<String> altTitle, AtomicBoolean keepGoing) {
if (!keepGoing.get()) {
return;
}
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
keepGoing.set(false);
return;
}
if (node.getNodeType() == Node.ELEMENT_NODE) {
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
Node item = node.getChildNodes().item(i);
findTitle(item, altTitle, keepGoing);
if (!keepGoing.get()) {
return;
}
}
} else if (node.getNodeType() == Node.TEXT_NODE) {
String text = node.getTextContent().trim();
if (!text.isBlank() && dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
altTitle.add(text);
}
}
}
}
@@ -0,0 +1,39 @@
package tn.mnlr.vripper.services.post;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.FailsafeException;
import org.apache.http.impl.execchain.RequestAbortedException;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
@Slf4j
public class MetadataRunnable implements Runnable {
@Getter
private final Post post;
private final MetadataCache metadataCache;
private final DataService dataService;
public MetadataRunnable(Post post) {
this.post = post;
this.metadataCache = SpringContext.getBean(MetadataCache.class);
this.dataService = SpringContext.getBean(DataService.class);
}
@Override
public void run() {
try {
Metadata metadata = metadataCache.get(post);
dataService.setMetadata(post, metadata);
} catch (Exception e) {
if (e.getCause() instanceof InterruptedException || (e.getCause() instanceof FailsafeException && (e.getCause().getCause() instanceof InterruptedException || e.getCause().getCause() instanceof RequestAbortedException))) {
return;
}
log.error(String.format("Failed to get metadata for postId %s", post.getPostId()), e);
}
}
}
@@ -0,0 +1,25 @@
package tn.mnlr.vripper.services.post;
import lombok.Getter;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import java.util.Optional;
import java.util.Set;
public class ParseResult {
private final Post post;
@Getter
private final Set<Image> images;
ParseResult(Post post, Set<Image> images) {
this.post = post;
this.images = images;
}
public Optional<Post> getPost() {
return Optional.ofNullable(post);
}
}
@@ -0,0 +1,85 @@
package tn.mnlr.vripper.services.post;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.q.PendingQ;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.CommonExecutor;
import tn.mnlr.vripper.services.DataService;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
@Service
@Slf4j
public class PostService {
private final AppSettingsService appSettingsService;
private final PendingQ pendingQ;
private final DataService dataService;
private final CommonExecutor commonExecutor;
private final Map<String, Future<?>> fetchingMetadata = new ConcurrentHashMap<>();
@Autowired
public PostService(AppSettingsService appSettingsService, PendingQ pendingQ, DataService dataService, DataService dataService1, CommonExecutor commonExecutor) {
this.appSettingsService = appSettingsService;
this.pendingQ = pendingQ;
this.dataService = dataService1;
this.commonExecutor = commonExecutor;
}
public void addPost(String postId, String threadId) throws PostParseException {
if (dataService.exists(postId)) {
log.warn(String.format("skipping %s, already loaded", postId));
return;
}
VRPostParser vrPostParser = new VRPostParser(threadId, postId);
ParseResult parseResult = vrPostParser.parse();
if (parseResult.getPost().isEmpty()) {
throw new PostParseException(String.format("parsing failed for thread %s, post %s", threadId, postId));
}
Post post = parseResult.getPost().get();
Set<Image> images = parseResult.getImages();
dataService.newPost(post, images);
// Metadata thread
fetchingMetadata.put(post.getPostId(), commonExecutor.getGeneralExecutor().submit(new MetadataRunnable(post)));
if (appSettingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
try {
pendingQ.enqueue(post, images);
} catch (InterruptedException e) {
log.warn("Interruption was caught");
Thread.currentThread().interrupt();
return;
}
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
log.debug("Auto start downloads option is disabled");
}
dataService.updatePostStatus(post.getStatus(), post.getId());
}
public void stopFetchingMetadata(Post post) {
this.fetchingMetadata.forEach((k, v) -> {
if (k.equals(post.getPostId())) {
v.cancel(true);
}
});
fetchingMetadata.remove(post.getPostId());
}
}
@@ -0,0 +1,108 @@
package tn.mnlr.vripper.services.post;
import lombok.extern.slf4j.Slf4j;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import java.util.Collection;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
class VRApiPostHandler extends DefaultHandler {
private final Collection<Host> supportedHosts;
private final String threadId;
private final String postId;
private final String postUrl;
private final Set<Image> images = new HashSet<>();
private Set<String> previews = new HashSet<>();
private String threadTitle;
private String postTitle;
private String forum;
private String userHash;
private int index = 0;
private Post parsedPost;
VRApiPostHandler(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
this.postUrl = String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", this.threadId, this.postId, this.postId);
this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
}
public ParseResult getParsedPost() {
return new ParseResult(parsedPost, images);
}
@Override
public void startDocument() {
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "forum":
forum = attributes.getValue("title").trim();
break;
case "user":
userHash = attributes.getValue("hash").trim();
break;
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
parsedPost = new Post(
postTitle,
postUrl,
postId,
threadId,
threadTitle,
forum,
userHash
);
break;
case "image":
index++;
if (previews.size() < 4) {
Optional.ofNullable(attributes.getValue("thumb_url")).map(String::trim).ifPresent(previews::add);
}
String mainUrl = Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
if (mainUrl != null) {
Host foundHost = supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst().orElse(null);
if (foundHost != null) {
log.debug(String.format("Found supported host %s for %s", foundHost.getClass().getSimpleName(), mainUrl));
images.add(new Image(postId, mainUrl, foundHost, index));
} else {
log.warn(String.format("unsupported host for %s, skipping", mainUrl));
}
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equals(qName.toLowerCase())) {
parsedPost.setTotal(images.size());
if (!previews.isEmpty()) {
parsedPost.setPreviews(previews);
}
parsedPost.setHosts(images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet()));
index = 0;
previews = new HashSet<>();
}
}
}
@@ -0,0 +1,79 @@
package tn.mnlr.vripper.services.post;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.VripperApplication;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import javax.xml.parsers.SAXParserFactory;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public class VRPostParser {
private static final String VR_API = "https://vipergirls.to/vr.php";
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final String threadId;
private final String postId;
private final ConnectionManager cm;
private final VipergirlsAuthService vipergirlsAuthService;
public VRPostParser(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
this.cm = SpringContext.getBean(ConnectionManager.class);
this.vipergirlsAuthService = SpringContext.getBean(VipergirlsAuthService.class);
}
public ParseResult parse() throws PostParseException {
log.debug(String.format("Parsing post %s", postId));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(VR_API);
uriBuilder.setParameter("p", postId);
httpGet = cm.buildHttpGet(uriBuilder.build());
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
AtomicReference<Throwable> thr = new AtomicReference<>();
VRApiPostHandler handler = new VRApiPostHandler(threadId, postId);
log.debug(String.format("Requesting %s", httpGet));
ParseResult post = getPost(httpGet, handler, thr);
if (thr.get() != null) {
log.error(String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
throw new PostParseException(thr.get());
}
return post;
}
private ParseResult getPost(HttpGet httpGet, VRApiPostHandler handler, AtomicReference<Throwable> thr) {
return Failsafe.with(VripperApplication.retryPolicy)
.onFailure(e -> thr.set(e.getFailure()))
.get(() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, vipergirlsAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
}
factory.newSAXParser().parse(response.getEntity().getContent(), handler);
EntityUtils.consumeQuietly(response.getEntity());
return handler.getParsedPost();
}
});
}
}
@@ -1,43 +0,0 @@
package tn.mnlr.vripper.web;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.web.socket.config.annotation.EnableWebSocket;
import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
import org.springframework.web.socket.server.standard.ServletServerContainerFactoryBean;
import tn.mnlr.vripper.web.wsendpoints.WebSocketHandler;
@Configuration
@EnableWebSocket
public class WebSocketConfig implements WebSocketConfigurer {
@Autowired
private WebSocketHandler handler;
@Override
public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
registry.addHandler(handler, "/endpoint").setAllowedOrigins("*");
}
@Bean
public TaskScheduler taskScheduler() {
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
scheduler.setPoolSize(2);
scheduler.setThreadNamePrefix("scheduled-task-");
scheduler.setDaemon(true);
return scheduler;
}
@Bean
public ServletServerContainerFactoryBean createWebSocketContainer() {
ServletServerContainerFactoryBean container = new ServletServerContainerFactoryBean();
container.setMaxSessionIdleTimeout(0L);
return container;
}
}
@@ -1,212 +0,0 @@
package tn.mnlr.vripper.web.restendpoints;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.CacheControl;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.AppStateExchange;
import tn.mnlr.vripper.services.PathService;
import tn.mnlr.vripper.services.ThumbnailGenerator;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import javax.servlet.http.HttpServletResponse;
import java.awt.*;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@RestController
@CrossOrigin(value = "*")
public class GalleryEndpoint {
private static final Logger logger = LoggerFactory.getLogger(GalleryEndpoint.class);
private static final String cacheControl = CacheControl.maxAge(365, TimeUnit.DAYS).getHeaderValue();
private final PathService pathService;
private final ThumbnailGenerator thumbnailGenerator;
private final AppSettingsService appSettingsService;
private final AppStateExchange appStateExchange;
@Autowired
public GalleryEndpoint(PathService pathService, ThumbnailGenerator thumbnailGenerator, AppSettingsService appSettingsService, AppStateExchange appStateExchange) {
this.pathService = pathService;
this.thumbnailGenerator = thumbnailGenerator;
this.appSettingsService = appSettingsService;
this.appStateExchange = appStateExchange;
}
@ExceptionHandler(Exception.class)
public ResponseEntity handleException(Exception e) {
logger.error("Error when process request", e);
return ResponseEntity
.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(e.getMessage());
}
@GetMapping(value = "/image/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
@ResponseStatus(value = HttpStatus.OK)
public ResponseEntity<byte[]> getImage(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
if (!appSettingsService.getSettings().getViewPhotos()) {
return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
}
response.addHeader("Cache-Control", cacheControl);
Post post = appStateExchange.getPost(postId);
File destinationFolder = pathService.getDownloadDestinationFolder(post);
return ResponseEntity.ok(Files.readAllBytes(Paths.get(destinationFolder.toPath().toString(), imgName)));
}
@GetMapping(value = "/image/thumb/{postId}/{imgName}", produces = {MediaType.IMAGE_JPEG_VALUE})
@ResponseStatus(value = HttpStatus.OK)
public ResponseEntity<byte[]> getImageThumb(@PathVariable("postId") @NonNull String postId, @PathVariable("imgName") @NonNull String imgName, HttpServletResponse response) throws Exception {
if (!appSettingsService.getSettings().getViewPhotos()) {
return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
}
response.addHeader("Cache-Control", cacheControl);
Post post = appStateExchange.getPost(postId);
File destinationFolder = pathService.getDownloadDestinationFolder(post);
if (destinationFolder.exists() && destinationFolder.isDirectory()) {
return ResponseEntity.ok(thumbnailGenerator.getThumbnails().get(new ThumbnailGenerator.CacheKey(postId, imgName)));
}
return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
}
@GetMapping("/gallery/{postId}")
@ResponseStatus(value = HttpStatus.OK)
public ResponseEntity<List<GalleryImage>> getGallery(@PathVariable("postId") @NonNull String postId) {
if (!appSettingsService.getSettings().getViewPhotos()) {
return new ResponseEntity("Gallery option is disabled", HttpStatus.BAD_REQUEST);
}
Post post = appStateExchange.getPost(postId);
File destinationFolder = pathService.getDownloadDestinationFolder(post);
if (destinationFolder.exists() && destinationFolder.isDirectory()) {
return ResponseEntity.ok(
Arrays.stream(Objects.requireNonNull(destinationFolder.listFiles()))
.filter(f -> !f.getName().endsWith("tmp"))
.filter(f -> f.getName().toLowerCase().endsWith(".jpg") || f.getName().toLowerCase().endsWith(".jpeg"))
.sorted(Comparator.comparing(File::getName))
.map(GalleryImage::fromFile)
.filter(Objects::nonNull)
.collect(Collectors.toList())
);
}
return new ResponseEntity("Gallery does not exist in download location, you probably removed it", HttpStatus.BAD_REQUEST);
}
@GetMapping("/gallery/cache")
@ResponseStatus(value = HttpStatus.OK)
public ResponseEntity<CacheSize> getCacheSize() {
return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
}
@GetMapping("/gallery/cache/clear")
@ResponseStatus(value = HttpStatus.OK)
public ResponseEntity<CacheSize> clearCache() {
thumbnailGenerator.clearCache();
return ResponseEntity.ok(new CacheSize(humanReadableByteCount(thumbnailGenerator.cacheSize(), false)));
}
private String humanReadableByteCount(long bytes, boolean si) {
int unit = si ? 1000 : 1024;
if (bytes < unit) return bytes + " B";
int exp = (int) (Math.log(bytes) / Math.log(unit));
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
}
@Getter
@Setter
@NoArgsConstructor
class CacheSize {
private String size;
public CacheSize(String size) {
this.size = size;
}
}
@Getter
@Setter
@NoArgsConstructor
class GalleryImage {
private static final CacheLoader<File, Dimension> loader = new CacheLoader<>() {
@Override
public Dimension load(File file) {
try (ImageInputStream in = ImageIO.createImageInputStream(file)) {
final Iterator<ImageReader> readers = ImageIO.getImageReaders(in);
if (readers.hasNext()) {
ImageReader reader = readers.next();
try {
reader.setInput(in);
return new Dimension(reader.getWidth(0), reader.getHeight(0));
} finally {
reader.dispose();
}
} else {
logger.error(String.format("No reader found for image %s", file.toString()));
return null;
}
} catch (Exception e) {
logger.error(String.format("Failed to create image object for %s", file.toString()), e);
return null;
}
}
};
private static final LoadingCache<File, Dimension> cache = CacheBuilder.newBuilder()
.maximumSize(20000)
.build(loader);
private static final Logger logger = LoggerFactory.getLogger(GalleryImage.class);
private String title;
private String src;
private String msrc;
private double w;
private double h;
public GalleryImage(String title, String src, String msrc, double w, double h) {
this.title = title;
this.src = src;
this.msrc = msrc;
this.w = w;
this.h = h;
}
public static GalleryImage fromFile(File file) {
Dimension dimension = null;
try {
dimension = cache.get(file);
} catch (ExecutionException e) {
logger.error(String.format("Failed to get image dimensions for %s", file.toString()), e);
}
if (dimension == null) {
return null;
}
return new GalleryImage(file.getName(), file.getName(), file.getName(), dimension.getWidth(), dimension.getHeight());
}
}
@@ -1,16 +1,21 @@
package tn.mnlr.vripper.web.restendpoints;
import lombok.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.entities.Post.METADATA;
import tn.mnlr.vripper.exception.PostParseException;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.q.ExecutionService;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.CommonExecutor;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PathService;
import tn.mnlr.vripper.services.VGHandler;
import tn.mnlr.vripper.services.post.CachedPost;
import tn.mnlr.vripper.services.post.PostService;
import tn.mnlr.vripper.web.restendpoints.domain.posts.*;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
@@ -19,178 +24,229 @@ import tn.mnlr.vripper.web.restendpoints.exceptions.ServerErrorException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Slf4j
@RestController
@CrossOrigin(value = "*")
public class PostRestEndpoint {
private static final Logger logger = LoggerFactory.getLogger(PostRestEndpoint.class);
private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?");
private final AppStateService appStateService;
private final AppStateExchange appStateExchange;
private final DataService dataService;
private final PathService pathService;
private final VGHandler vgHandler;
private final ExecutionService executionService;
private final PostParser postParser;
private final PostService postService;
private final CommonExecutor commonExecutor;
private static final Byte LOCK = -1;
@Autowired
public PostRestEndpoint(AppStateService appStateService, AppStateExchange appStateExchange, PathService pathService, VGHandler vgHandler, ExecutionService executionService, PostParser postParser, CommonExecutor commonExecutor) {
this.appStateService = appStateService;
this.appStateExchange = appStateExchange;
public PostRestEndpoint(DataService dataService, PathService pathService, VGHandler vgHandler, ExecutionService executionService, PostService postService, CommonExecutor commonExecutor) {
this.dataService = dataService;
this.pathService = pathService;
this.vgHandler = vgHandler;
this.executionService = executionService;
this.postParser = postParser;
this.postService = postService;
this.commonExecutor = commonExecutor;
}
@PostMapping("/post")
@ResponseStatus(code = HttpStatus.OK)
public void processPost(@RequestBody ThreadUrl _url) {
if (_url.getUrl() == null || _url.getUrl().isEmpty()) {
throw new BadRequestException("Failed to process empty request");
}
List<String> urls = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
ArrayList<QueuedVGLink> queuedVGLinks = new ArrayList<>();
for (String url : urls) {
logger.debug(String.format("Starting to process thread: %s", url));
if (!url.startsWith("https://vipergirls.to")) {
logger.error(String.format("Unsupported link %s", url));
continue;
synchronized (LOCK) {
if (_url.getUrl() == null || _url.getUrl().isBlank()) {
log.error("Cannot process empty requests");
throw new BadRequestException("Cannot process empty requests");
}
List<String> urlList = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
ArrayList<Queued> queuedList = new ArrayList<>();
for (String url : urlList) {
log.debug(String.format("Starting to process thread: %s", url));
if (!url.startsWith("https://vipergirls.to")) {
log.error(String.format("Unsupported link %s", url));
continue;
}
String threadId, postId;
Matcher m = VG_URL_PATTERN.matcher(url);
if (m.find()) {
threadId = m.group(1);
postId = m.group(4);
} else {
throw new BadRequestException(String.format("Cannot retrieve thread id from URL %s", url));
String threadId, postId;
Matcher m = VG_URL_PATTERN.matcher(url);
if (m.find()) {
threadId = m.group(1);
postId = m.group(4);
} else {
throw new BadRequestException(String.format("Cannot retrieve thread id from URL %s", url));
}
queuedList.add(new Queued(url, threadId, postId));
}
try {
vgHandler.handle(queuedList);
} catch (Exception e) {
log.error("Failed to parse links", e);
throw new ServerErrorException(e.getMessage());
}
queuedVGLinks.add(new QueuedVGLink(url, threadId, postId));
}
try {
vgHandler.handle(queuedVGLinks);
} catch (Exception e) {
throw new ServerErrorException(e.getMessage());
}
}
@PostMapping("/post/restart")
@ResponseStatus(value = HttpStatus.OK)
public void restartPost(@RequestBody @NonNull List<PostId> postIds) {
executionService.restartAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
synchronized (LOCK) {
executionService.restartAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
}
}
@PostMapping("/post/add")
@ResponseStatus(value = HttpStatus.OK)
public void addPost(@RequestBody List<PostToAdd> posts) {
for (PostToAdd post : posts) {
commonExecutor.getGeneralExecutor().submit(() -> {
try {
postParser.addPost(post.getPostId(), post.getThreadId());
} catch (PostParseException e) {
logger.error(String.format("Failed to add post %s", post.getPostId()), e);
}
});
synchronized (LOCK) {
for (PostToAdd post : posts) {
commonExecutor.getGeneralExecutor().submit(() -> {
try {
postService.addPost(post.getPostId(), post.getThreadId());
} catch (PostParseException e) {
log.error(String.format("Failed to add post %s", post.getPostId()), e);
throw new ServerErrorException(String.format("Failed to add post %s", post.getPostId()));
}
});
}
}
}
@GetMapping("/post/path/{postId}")
@ResponseStatus(value = HttpStatus.OK)
public DownloadPath folderPath(@PathVariable("postId") String postId) {
Post post = appStateExchange.getPost(postId);
if (post.getPostFolderName() == null) {
throw new NotFoundException("Download has not been started yet for this post");
synchronized (LOCK) {
return getDownloadPath(postId);
}
}
private DownloadPath getDownloadPath(String postId) {
Optional<Post> _post = dataService.findPostByPostId(postId);
if (_post.isPresent()) {
Post post = _post.get();
if (post.getPostFolderName() == null) {
log.error("Download has not been started yet for this post");
throw new NotFoundException("Download has not been started yet for this post");
} else {
return new DownloadPath(pathService.getDownloadDestinationFolder(post).getPath());
}
} else {
return new DownloadPath(pathService.getDownloadDestinationFolder(post).getPath());
log.error(String.format("Unable to find post with postId = %s", postId));
throw new NotFoundException(String.format("Unable to find post with postId = %s", postId));
}
}
@PostMapping("/post/restart/all")
@ResponseStatus(value = HttpStatus.OK)
public void restartPost() {
executionService.restartAll(null);
synchronized (LOCK) {
executionService.restartAll(null);
}
}
@PostMapping("/post/stop")
@ResponseStatus(value = HttpStatus.OK)
public void stop(@RequestBody @NonNull List<PostId> postIds) {
executionService.stopAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
synchronized (LOCK) {
executionService.stopAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
}
}
@PostMapping("/post/stop/all")
@ResponseStatus(value = HttpStatus.OK)
public void stopAll() {
executionService.stopAll(null);
synchronized (LOCK) {
executionService.stopAll(null);
}
}
@PostMapping("/post/remove")
@ResponseStatus(value = HttpStatus.OK)
public List<RemoveResult> remove(@RequestBody @NonNull List<PostId> postIds) {
List<RemoveResult> result = new ArrayList<>();
List<String> collect = postIds.stream().map(PostId::getPostId).peek(e -> result.add(new RemoveResult(e))).collect(Collectors.toList());
executionService.stopAll(collect);
appStateService.removeAll(collect);
return result;
synchronized (LOCK) {
List<RemoveResult> result = new ArrayList<>();
List<String> collect = postIds.stream().map(PostId::getPostId).peek(e -> result.add(new RemoveResult(e))).collect(Collectors.toList());
executionService.stopAll(collect);
dataService.removeAll(collect);
return result;
}
}
@PostMapping("/post/rename")
@ResponseStatus(value = HttpStatus.OK)
public List<AltPostName> rename(@RequestBody @NonNull List<AltPostName> postToRename) {
synchronized (LOCK) {
renamePosts(postToRename);
return postToRename;
}
}
private void renamePosts(@RequestBody @NonNull List<AltPostName> postToRename) {
for (AltPostName altPostName : postToRename) {
Post post = appStateExchange.getPost(altPostName.getPostId());
try {
pathService.rename(post, altPostName.getAltName());
pathService.rename(altPostName.getPostId(), altPostName.getAltName());
} catch (Exception e) {
log.error(String.format("Failed to rename post with postId = %s", altPostName.getPostId()), e);
throw new ServerErrorException(e.getMessage());
}
}
return postToRename;
}
@PostMapping("/post/rename/first")
@ResponseStatus(value = HttpStatus.OK)
public List<PostId> renameFirst(@RequestBody @NonNull List<PostId> postToRename) {
for (PostId postId : postToRename) {
Post post = appStateExchange.getPost(postId.getPostId());
if (post.getMetadata().get(METADATA.RESOLVED_NAME.name()) != null) {
List<String> resolvedNames = ((List<String>) post.getMetadata().get(METADATA.RESOLVED_NAME.name()));
if (!resolvedNames.isEmpty()) {
try {
pathService.rename(post, resolvedNames.get(0));
} catch (Exception e) {
throw new ServerErrorException(e.getMessage());
synchronized (LOCK) {
renamePostsToFirst(postToRename);
return postToRename;
}
}
public void renamePostsToFirst(@RequestBody @NonNull List<PostId> postToRename) {
synchronized (LOCK) {
for (PostId postId : postToRename) {
Optional<Metadata> _metadata = dataService.findMetadataByPostId(postId.getPostId());
if (_metadata.isPresent()) {
Metadata metadata = _metadata.get();
if (metadata.getResolvedNames() != null) {
List<String> resolvedNames = metadata.getResolvedNames();
if (!resolvedNames.isEmpty()) {
String altTitle = resolvedNames.get(0);
try {
pathService.rename(postId.getPostId(), altTitle);
} catch (Exception e) {
log.error(String.format("Failed to rename post with postId = %s", postId.getPostId()), e);
throw new ServerErrorException(e.getMessage());
}
}
}
}
}
}
return postToRename;
}
@PostMapping("/post/clear/all")
@ResponseStatus(value = HttpStatus.OK)
public RemoveAllResult clearAll() {
return new RemoveAllResult(appStateService.clearAll());
}
@PostMapping("/post/remove/all")
@ResponseStatus(value = HttpStatus.OK)
public RemoveAllResult removeAll() {
return new RemoveAllResult(appStateService.removeAll(null));
synchronized (LOCK) {
return new RemoveAllResult(dataService.clearCompleted());
}
}
@GetMapping("/grab/{threadId}")
@ResponseStatus(value = HttpStatus.OK)
public List<VRPostState> grab(@PathVariable("threadId") @NonNull String threadId) throws Exception {
QueuedVGLink queuedVGLink = appStateExchange.getQueue().values().stream().filter(e -> e.getThreadId().equals(threadId)).findFirst().orElseThrow();
return vgHandler.getCache().get(queuedVGLink);
public List<CachedPost> grab(@PathVariable("threadId") @NonNull String threadId) {
Queued queued = dataService.findQueuedByThreadId(threadId).orElseThrow(() -> new NotFoundException(String.format("Unable to find links for threadId = %s", threadId)));
try {
return vgHandler.getCache().get(queued);
} catch (ExecutionException e) {
log.error(String.format("Failed to get links for threadId = %s", threadId), e);
throw new ServerErrorException(String.format("Failed to get links for threadId = %s", threadId));
}
}
@PostMapping("/grab/remove")
@@ -1,5 +1,6 @@
package tn.mnlr.vripper.web.restendpoints;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
@@ -9,6 +10,7 @@ import tn.mnlr.vripper.services.VipergirlsAuthService;
import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
@RestController
@Slf4j
@CrossOrigin(value = "*")
public class SettingsRestEndpoint {
@@ -24,10 +26,8 @@ public class SettingsRestEndpoint {
@PostMapping("/settings/theme")
@ResponseStatus(value = HttpStatus.OK)
public AppSettingsService.Theme postTheme(@RequestBody AppSettingsService.Theme theme) {
synchronized (this.appSettingsService) {
this.appSettingsService.setTheme(theme);
return appSettingsService.getTheme();
}
this.appSettingsService.setTheme(theme);
return appSettingsService.getTheme();
}
@GetMapping("/settings/theme")
@@ -40,16 +40,15 @@ public class SettingsRestEndpoint {
@ResponseStatus(value = HttpStatus.OK)
public AppSettingsService.Settings postSettings(@RequestBody AppSettingsService.Settings settings) throws Exception {
synchronized (this.appSettingsService) {
try {
this.appSettingsService.check(settings);
} catch (ValidationException e) {
throw new BadRequestException(e.getMessage());
}
this.appSettingsService.newSettings(settings);
vipergirlsAuthService.authenticate();
try {
this.appSettingsService.check(settings);
} catch (ValidationException e) {
log.error("Invalid settings", e);
throw new BadRequestException(e.getMessage());
}
this.appSettingsService.newSettings(settings);
vipergirlsAuthService.authenticate();
return getAppSettingsService();
}
@@ -0,0 +1,81 @@
package tn.mnlr.vripper.web.wsendpoints;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.handler.annotation.DestinationVariable;
import org.springframework.messaging.simp.annotation.SubscribeMapping;
import org.springframework.stereotype.Controller;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.services.*;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@Controller
public class AppDataController {
private final VipergirlsAuthService vipergirlsAuthService;
private final GlobalStateService globalStateService;
private final DownloadSpeedService downloadSpeedService;
private final DataService dataService;
private final PathService pathService;
@Autowired
public AppDataController(VipergirlsAuthService vipergirlsAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService, PathService pathService) {
this.vipergirlsAuthService = vipergirlsAuthService;
this.globalStateService = globalStateService;
this.downloadSpeedService = downloadSpeedService;
this.dataService = dataService;
this.pathService = pathService;
}
@Getter
public static class LoggedUser {
private final String user;
LoggedUser(String user) {
this.user = user;
}
}
@SubscribeMapping("/user")
public LoggedUser user() {
return new LoggedUser(vipergirlsAuthService.getLoggedUser());
}
@SubscribeMapping("/download-state")
public GlobalState downloadState() {
return globalStateService.getCurrentState();
}
@SubscribeMapping("/speed")
public DownloadSpeed speed() {
return new DownloadSpeed(downloadSpeedService.getCurrentValue());
}
@SubscribeMapping("/posts")
public Collection<Post> posts() {
return StreamSupport.stream(dataService.findAllPosts().spliterator(), false).peek(this::isRenaming).collect(Collectors.toList());
}
@SubscribeMapping("/images/{postId}")
public List<Image> postsDetails(@DestinationVariable("postId") String postId) {
return dataService.findImagesByPostId(postId);
}
@SubscribeMapping("/queued")
public Collection<Queued> queued() {
return StreamSupport.stream(dataService.findAllQueued().spliterator(), false).collect(Collectors.toList());
}
private void isRenaming(Post post) {
if (pathService.getRenaming().contains(post.getPostId())) {
post.setRenaming(true);
}
}
}
@@ -0,0 +1,121 @@
package tn.mnlr.vripper.web.wsendpoints;
import io.reactivex.disposables.Disposable;
import io.reactivex.schedulers.Schedulers;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.*;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
@Slf4j
public class WebSocketBroadcast {
private final SimpMessagingTemplate template;
private final VipergirlsAuthService vipergirlsAuthService;
private final GlobalStateService globalStateService;
private final DownloadSpeedService downloadSpeedService;
private final DataService dataService;
private final PathService pathService;
private final List<Disposable> disposables = new ArrayList<>();
@Autowired
public WebSocketBroadcast(SimpMessagingTemplate template, VipergirlsAuthService vipergirlsAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService, PathService pathService) {
this.template = template;
this.vipergirlsAuthService = vipergirlsAuthService;
this.globalStateService = globalStateService;
this.downloadSpeedService = downloadSpeedService;
this.dataService = dataService;
this.pathService = pathService;
}
@PostConstruct
private void run() {
disposables.add(vipergirlsAuthService.getLoggedInUser()
.subscribeOn(Schedulers.io())
.onBackpressureBuffer()
.map(AppDataController.LoggedUser::new)
.subscribe(user -> template.convertAndSend("/topic/user", user), e -> log.error("Failed to send data to client", e)));
disposables.add(globalStateService.getLiveGlobalState()
.subscribeOn(Schedulers.io())
.onBackpressureBuffer()
.subscribe(state -> template.convertAndSend("/topic/download-state", state), e -> log.error("Failed to send data to client", e)));
disposables.add(downloadSpeedService.getReadBytesPerSecond()
.subscribeOn(Schedulers.io())
.onBackpressureBuffer()
.map(DownloadSpeed::new)
.subscribe(speed -> template.convertAndSend("/topic/speed", speed), e -> log.error("Failed to send data to client", e)));
disposables.add(dataService.livePost()
.subscribeOn(Schedulers.io())
.buffer(500, TimeUnit.MILLISECONDS)
.map(HashSet::new)
.filter(e -> !e.isEmpty())
.subscribe(ids -> template.convertAndSend("/topic/posts", ids.stream().map(dataService::findPostById).filter(Optional::isPresent).map(Optional::get).peek(this::isRenaming).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e)));
disposables.add(dataService.liveImage()
.subscribeOn(Schedulers.io())
.buffer(500, TimeUnit.MILLISECONDS)
.map(HashSet::new)
.filter(e -> !e.isEmpty())
.subscribe(id -> id
.stream()
.map(dataService::findImageById)
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.groupingBy(Image::getPostId))
.forEach((postId, images) -> template.convertAndSend("/topic/images/" + postId, images)),
e -> log.error("Failed to send data to client", e))
);
disposables.add(dataService.liveQueue()
.subscribeOn(Schedulers.io())
.buffer(500, TimeUnit.MILLISECONDS)
.map(HashSet::new)
.filter(e -> !e.isEmpty())
.subscribe(ids -> template.convertAndSend("/topic/queued", ids.stream().map(dataService::findQueuedById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e))
);
disposables.add(dataService.queueRemove()
.subscribeOn(Schedulers.io())
.buffer(500, TimeUnit.MILLISECONDS)
.map(HashSet::new)
.filter(e -> !e.isEmpty())
.subscribe(threadIds -> template.convertAndSend("/topic/queued/deleted", threadIds), e -> log.error("Failed to send data to client", e))
);
disposables.add(dataService.postRemove()
.subscribeOn(Schedulers.io())
.buffer(500, TimeUnit.MILLISECONDS)
.map(HashSet::new)
.filter(e -> !e.isEmpty())
.subscribe(postIds -> template.convertAndSend("/topic/posts/deleted", postIds), e -> log.error("Failed to send data to client", e))
);
}
private void isRenaming(Post post) {
if (pathService.getRenaming().contains(post.getPostId())) {
post.setRenaming(true);
}
}
@PreDestroy
private void destroy() {
disposables.forEach(Disposable::dispose);
}
}
@@ -0,0 +1,24 @@
package tn.mnlr.vripper.web.wsendpoints;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
@Configuration
@EnableWebSocketMessageBroker
public class WebSocketConfig implements WebSocketMessageBrokerConfigurer {
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/ws")
.setAllowedOrigins("*");
}
@Override
public void configureMessageBroker(MessageBrokerRegistry registry) {
registry.setApplicationDestinationPrefixes("/app", "/topic");
registry.enableSimpleBroker("/topic");
}
}
@@ -1,327 +0,0 @@
package tn.mnlr.vripper.web.wsendpoints;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.reactivex.disposables.Disposable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.TextWebSocketHandler;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.entities.mixin.ui.ImageUIMixin;
import tn.mnlr.vripper.entities.mixin.ui.PostUIMixin;
import tn.mnlr.vripper.services.*;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Component
public class WebSocketHandler extends TextWebSocketHandler {
private static final Logger logger = LoggerFactory.getLogger(WebSocketHandler.class);
private final GlobalStateService globalStateService;
private final AppStateExchange appStateExchange;
private final DownloadSpeedService downloadSpeedService;
private final VipergirlsAuthService vipergirlsAuthService;
private final CommonExecutor commonExecutor;
private final Map<String, Disposable> postsSubscriptions = new ConcurrentHashMap<>();
private final Map<String, Disposable> postDetailsSubscriptions = new ConcurrentHashMap<>();
private final Map<String, Disposable> stateSubscriptions = new ConcurrentHashMap<>();
private final Map<String, Disposable> downloadSpeedSubscriptions = new ConcurrentHashMap<>();
private final Map<String, Disposable> userSubscriptions = new ConcurrentHashMap<>();
private final Map<String, Disposable> grabQueueSubscriptions = new ConcurrentHashMap<>();
private final Map<String, Future<Void>> threadParseRequests = new ConcurrentHashMap<>();
private final ObjectMapper om = new ObjectMapper();
@Autowired
public WebSocketHandler(GlobalStateService globalStateService, AppStateExchange appStateExchange, DownloadSpeedService downloadSpeedService, VipergirlsAuthService vipergirlsAuthService, CommonExecutor commonExecutor) {
this.globalStateService = globalStateService;
this.appStateExchange = appStateExchange;
this.downloadSpeedService = downloadSpeedService;
this.vipergirlsAuthService = vipergirlsAuthService;
this.commonExecutor = commonExecutor;
om.addMixIn(Image.class, ImageUIMixin.class).addMixIn(Post.class, PostUIMixin.class);
}
@Override
protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception {
WSMessage wsMessage = om.readValue(message.getPayload(), WSMessage.class);
WSMessage.CMD cmd = WSMessage.CMD.valueOf(wsMessage.getCmd());
switch (cmd) {
case GRAB_QUEUE_SUB:
subscribeForGrabQueue(session);
break;
case GLOBAL_STATE_SUB:
subscribeForGlobalState(session);
break;
case SPEED_SUB:
subscribeForSpeed(session);
break;
case USER_SUB:
subscribeForUser(session);
break;
case POSTS_SUB:
subscribeForPosts(session);
break;
case POST_DETAILS_SUB:
subscribeForPostDetails(session, wsMessage.getPayload());
break;
case POST_DETAILS_UNSUB:
logger.debug(String.format("Client %s unsubscribed from post details", session.getId()));
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
break;
case POSTS_UNSUB:
logger.debug(String.format("Client %s unsubscribed from posts", session.getId()));
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
break;
case GLOBAL_STATE_UNSUB:
logger.debug(String.format("Client %s unsubscribed from global state", session.getId()));
Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
break;
case SPEED_UNSUB:
logger.debug(String.format("Client %s unsubscribed from download speed info", session.getId()));
Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
break;
case USER_UNSUB:
logger.debug(String.format("Client %s unsubscribed from user info", session.getId()));
Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
break;
case GRAB_QUEUE_UNSUB:
logger.debug(String.format("Client %s unsubscribed from grab queue", session.getId()));
Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
break;
}
}
private void subscribeForGlobalState(WebSocketSession session) {
logger.debug(String.format("Client %s subscribed for global state", session.getId()));
if (stateSubscriptions.containsKey(session.getId())) {
stateSubscriptions.get(session.getId()).dispose();
}
try {
send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(globalStateService.getCurrentState()))));
} catch (Exception e) {
logger.error("Unexpected error occurred", e);
}
stateSubscriptions.put(session.getId(),
globalStateService.getLiveGlobalState()
.subscribeOn(commonExecutor.getScheduler())
.onBackpressureBuffer()
.map(Collections::singleton)
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private void subscribeForSpeed(WebSocketSession session) {
logger.debug(String.format("Client %s subscribed for download speed info", session.getId()));
if (downloadSpeedSubscriptions.containsKey(session.getId())) {
downloadSpeedSubscriptions.get(session.getId()).dispose();
}
try {
send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new DownloadSpeed(0)))));
} catch (Exception e) {
logger.error("Unexpected error occurred", e);
}
downloadSpeedSubscriptions.put(session.getId(),
downloadSpeedService.getReadBytesPerSecond()
.subscribeOn(commonExecutor.getScheduler())
.onBackpressureBuffer()
.map(e -> Collections.singleton(new DownloadSpeed(e)))
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private void subscribeForUser(WebSocketSession session) {
logger.debug(String.format("Client %s subscribed for user info", session.getId()));
if (userSubscriptions.containsKey(session.getId())) {
userSubscriptions.get(session.getId()).dispose();
}
try {
send(session, new TextMessage(om.writeValueAsString(Collections.singletonList(new LoggedUser(vipergirlsAuthService.getLoggedUser())))));
} catch (Exception e) {
logger.error("Unexpected error occurred", e);
}
userSubscriptions.put(session.getId(),
vipergirlsAuthService.getLoggedInUser()
.subscribeOn(commonExecutor.getScheduler())
.onBackpressureBuffer()
.map(e -> Collections.singleton(new LoggedUser(e)))
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private void subscribeForPosts(WebSocketSession session) {
logger.debug(String.format("Client %s subscribed for posts", session.getId()));
if (postsSubscriptions.containsKey(session.getId())) {
postsSubscriptions.get(session.getId()).dispose();
}
try {
send(session, new TextMessage(om.writeValueAsString(appStateExchange.getPosts().values())));
} catch (Exception e) {
logger.error("Unexpected error occurred", e);
}
postsSubscriptions.put(session.getId(),
appStateExchange.livePost()
.subscribeOn(commonExecutor.getScheduler())
.onBackpressureBuffer()
.buffer(2000, TimeUnit.MILLISECONDS, 200)
.filter(e -> !e.isEmpty())
.map(HashSet::new)
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private void subscribeForGrabQueue(WebSocketSession session) {
logger.debug(String.format("Client %s subscribed for grab queue", session.getId()));
if (grabQueueSubscriptions.containsKey(session.getId())) {
grabQueueSubscriptions.get(session.getId()).dispose();
}
try {
send(session, new TextMessage(om.writeValueAsString(appStateExchange.getQueue().values())));
} catch (Exception e) {
logger.error("Unexpected error occurred", e);
}
grabQueueSubscriptions.put(session.getId(),
appStateExchange.liveQueue()
.subscribeOn(commonExecutor.getScheduler())
.onBackpressureBuffer()
.buffer(2000, TimeUnit.MILLISECONDS, 200)
.filter(e -> !e.isEmpty())
.map(HashSet::new)
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private void subscribeForPostDetails(WebSocketSession session, String postId) {
logger.debug(String.format("Client %s subscribed for post details with id = %s", session.getId(), postId));
if (postDetailsSubscriptions.containsKey(session.getId())) {
postDetailsSubscriptions.get(session.getId()).dispose();
}
try {
send(session, new TextMessage(om.writeValueAsString(
appStateExchange.getImages()
.values()
.stream()
.filter(e -> e.getPostId().equals(postId))
.collect(Collectors.toSet())))
);
} catch (Exception e) {
logger.error("Unexpected error occurred", e);
}
postDetailsSubscriptions.put(session.getId(), appStateExchange.liveImage()
.subscribeOn(commonExecutor.getScheduler())
.onBackpressureBuffer()
.filter(e -> e.getPostId().equals(postId))
.buffer(2000, TimeUnit.MILLISECONDS, 500)
.filter(e -> !e.isEmpty())
.map(HashSet::new)
.map(om::writeValueAsString)
.map(TextMessage::new)
.subscribe(msg -> send(session, msg), e -> logger.error("Failed to send data to client", e))
);
}
private synchronized void send(WebSocketSession session, TextMessage message) throws IOException {
session.sendMessage(message);
}
@Override
public void afterConnectionEstablished(WebSocketSession session) {
logger.debug(String.format("Connection open for client id: %s", session.getId()));
}
@Override
public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
Optional.ofNullable(postsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
Optional.ofNullable(postDetailsSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
Optional.ofNullable(stateSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
Optional.ofNullable(downloadSpeedSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
Optional.ofNullable(userSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
Optional.ofNullable(grabQueueSubscriptions.remove(session.getId())).ifPresent(Disposable::dispose);
Optional.ofNullable(threadParseRequests.remove(session.getId())).ifPresent(d -> d.cancel(true));
logger.debug(String.format("Connection closed for client id: %s", session.getId()));
}
@Getter
@Setter
@NoArgsConstructor
private static class WSMessage {
private String cmd;
private String payload;
enum CMD {
POSTS_SUB,
POST_DETAILS_SUB,
POSTS_UNSUB,
POST_DETAILS_UNSUB,
GLOBAL_STATE_SUB,
GLOBAL_STATE_UNSUB,
SPEED_SUB,
SPEED_UNSUB,
USER_SUB,
USER_UNSUB,
GRAB_QUEUE_SUB,
GRAB_QUEUE_UNSUB
}
}
@Getter
private static class LoggedUser {
private final String type = "user";
private String user;
LoggedUser(String user) {
this.user = user;
}
}
}
@@ -1 +0,0 @@
base.dir=${user.home}
@@ -1 +0,0 @@
base.dir=${user.dir}
@@ -1,9 +1,16 @@
logging.level.org.springframework.web=INFO
logging.level.root=INFO
logging.level.org.apache.http=INFO
logging.file=${base.dir}/.vripper/vripper.log
logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats=ERROR
logging.file.name=${base.dir}/${base.dir.name}/vripper.log
server.port=${vripper.server.port:8080}
management.endpoints.web.exposure.include=shutdown
management.endpoint.shutdown.enabled=true
spring.profiles.active=portable
#spring.profiles.active=installer
server.error.include-message=always
spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.xml
spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/xparty;hsqldb.lock_file=false
spring.datasource.username=SA
spring.datasource.password=lEtmEIn
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
base.dir=${user.dir}
base.dir.name=${vripper.base.dir.name:vripper}
@@ -0,0 +1,167 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
<changeSet author="sysgen" id="1595764509827-1">
<createSequence sequenceName="HIBERNATE_SEQUENCE"/>
</changeSet>
<changeSet author="sysgen" id="1595764509827-2">
<createTable tableName="IMAGE">
<column name="ID" type="BIGINT">
<constraints nullable="false"/>
</column>
<column name="CURRENT" type="BIGINT">
<constraints nullable="false"/>
</column>
<column name="HOST" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
<column name="INDEX" type="INT">
<constraints nullable="false"/>
</column>
<column name="POST_ID" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
<column name="STATUS" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
<column name="TOTAL" type="BIGINT">
<constraints nullable="false"/>
</column>
<column name="URL" type="VARCHAR(3000)">
<constraints nullable="false"/>
</column>
<column name="POST_ID_REF" type="BIGINT"/>
</createTable>
</changeSet>
<changeSet author="sysgen" id="1595764509827-3">
<createTable tableName="METADATA">
<column name="ID" type="BIGINT">
<constraints nullable="false"/>
</column>
<column name="POSTED_BY" type="VARCHAR(255)"/>
<column name="RESOLVED_NAMES" type="VARCHAR(16777216)"/>
<column name="POST_ID_REF" type="BIGINT">
<constraints nullable="false"/>
</column>
</createTable>
</changeSet>
<changeSet author="sysgen" id="1595764509827-4">
<createTable tableName="POST">
<column name="ID" type="BIGINT">
<constraints nullable="false"/>
</column>
<column name="DONE" type="INT">
<constraints nullable="false"/>
</column>
<column name="FORUM" type="VARCHAR(500)">
<constraints nullable="false"/>
</column>
<column name="HOSTS" type="VARCHAR(500)">
<constraints nullable="false"/>
</column>
<column name="POST_FOLDER_NAME" type="VARCHAR(500)"/>
<column name="POST_ID" type="VARCHAR(255)"/>
<column name="PREVIEWS" type="VARCHAR(16777216)"/>
<column name="SECURITY_TOKEN" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
<column name="STATUS" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
<column name="THANKED" type="BOOLEAN"/>
<column name="THREAD_ID" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
<column name="THREAD_TITLE" type="VARCHAR(500)"/>
<column name="TITLE" type="VARCHAR(500)">
<constraints nullable="false"/>
</column>
<column name="TOTAL" type="INT">
<constraints nullable="false"/>
</column>
<column name="URL" type="VARCHAR(3000)">
<constraints nullable="false"/>
</column>
</createTable>
</changeSet>
<changeSet author="sysgen" id="1595764509827-5">
<createTable tableName="QUEUED">
<column name="ID" type="BIGINT">
<constraints nullable="false"/>
</column>
<column name="TOTAL" type="INT">
<constraints nullable="false"/>
</column>
<column name="LINK" type="VARCHAR(3000)">
<constraints nullable="false"/>
</column>
<column name="LOADING" type="BOOLEAN">
<constraints nullable="false"/>
</column>
<column name="POST_ID" type="VARCHAR(255)"/>
<column name="THREAD_ID" type="VARCHAR(255)">
<constraints nullable="false"/>
</column>
</createTable>
</changeSet>
<changeSet author="sysgen" id="1595764509827-6">
<addPrimaryKey columnNames="ID" constraintName="SYS_PK_10092" tableName="IMAGE"/>
</changeSet>
<changeSet author="sysgen" id="1595764509827-7">
<addPrimaryKey columnNames="ID" constraintName="SYS_PK_10110" tableName="METADATA"/>
</changeSet>
<changeSet author="sysgen" id="1595764509827-8">
<addPrimaryKey columnNames="ID" constraintName="SYS_PK_10116" tableName="POST"/>
</changeSet>
<changeSet author="sysgen" id="1595764509827-9">
<addPrimaryKey columnNames="ID" constraintName="SYS_PK_10138" tableName="QUEUED"/>
</changeSet>
<changeSet author="sysgen" id="1595764509827-10">
<createIndex indexName="IMAGE_POST_ID_IDX" tableName="IMAGE">
<column name="POST_ID"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-11">
<createIndex indexName="IMAGE_STATUS_IDX" tableName="IMAGE">
<column name="STATUS"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-12">
<createIndex indexName="POST_POST_ID_IDX" tableName="POST">
<column name="POST_ID"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-13">
<createIndex indexName="POST_STATUS_IDX" tableName="POST">
<column name="STATUS"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-14">
<createIndex indexName="QUEUED_THREAD_ID_IDX" tableName="QUEUED">
<column name="THREAD_ID"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-15">
<createIndex indexName="SYS_IDX_IMAGE_POST_ID_REF_POST_ID_FK_10150" tableName="IMAGE">
<column name="POST_ID_REF"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-16">
<createIndex indexName="SYS_IDX_METADATA_POST_ID_REF_POST_ID_FK_10160" tableName="METADATA">
<column name="POST_ID_REF"/>
</createIndex>
</changeSet>
<changeSet author="sysgen" id="1595764509827-17">
<addForeignKeyConstraint baseColumnNames="POST_ID_REF" baseTableName="IMAGE"
constraintName="IMAGE_POST_ID_REF_POST_ID_FK" deferrable="false"
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
referencedColumnNames="ID" referencedTableName="POST"/>
</changeSet>
<changeSet author="sysgen" id="1595764509827-18">
<addForeignKeyConstraint baseColumnNames="POST_ID_REF" baseTableName="METADATA"
constraintName="METADATA_POST_ID_REF_POST_ID_FK" deferrable="false"
initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
referencedColumnNames="ID" referencedTableName="POST"/>
</changeSet>
</databaseChangeLog>
+4
View File
@@ -0,0 +1,4 @@
{
"repoId": "65cf9772-12a2-4386-8124-a1813f7a18be",
"lastSync": 0
}
+1 -1
View File
@@ -10,7 +10,7 @@
"prefix": "app",
"schematics": {
"@schematics/angular:component": {
"styleext": "scss"
"style": "scss"
}
},
"architect": {
+1 -1
View File
@@ -1,5 +1,5 @@
{
"extends": "../tsconfig.json",
"extends": "../tsconfig.base.json",
"compilerOptions": {
"outDir": "../out-tsc/app",
"module": "commonjs",

Some files were not shown because too many files have changed in this diff Show More