Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6375c0aed6 | ||
|
|
40ca4cf86b | ||
|
|
b3a798388b | ||
|
|
6e69b2bce8 | ||
|
|
bb6dcd77b9 | ||
|
|
42e1aa73e2 | ||
|
|
26fde22af2 | ||
|
|
798f39d54f | ||
|
|
3a21dfac38 | ||
|
|
e8355b88b9 | ||
|
|
379660236d | ||
|
|
c86ee91c27 | ||
|
|
ff9d253242 | ||
|
|
53b45e00ba | ||
|
|
247b12a525 | ||
|
|
7a8d332e6e | ||
|
|
749a6f18d2 | ||
|
|
4107048f12 | ||
|
|
f5d9ab85a2 | ||
|
|
b82ee7b494 | ||
|
|
8a31dfc012 | ||
|
|
734c1b72c7 | ||
|
|
7abb7df1f6 | ||
|
|
0a7fa99a64 | ||
|
|
691026c8a9 | ||
|
|
3ef004b947 | ||
|
|
1b22a93e28 | ||
|
|
2639df0aaa | ||
|
|
d5ae5d57d1 | ||
|
|
537d76d7be | ||
|
|
c1247faacc | ||
|
|
efcc685f30 | ||
|
|
c825b22ce4 | ||
|
|
6ef5d5474f | ||
|
|
bd25801c6c | ||
|
|
377b93c1ac | ||
|
|
d498d20f6b | ||
|
|
d18544d414 | ||
|
|
3c872b79bd | ||
|
|
37900cac49 | ||
|
|
be57c76d93 | ||
|
|
8b52ad7858 | ||
|
|
a0976bde02 | ||
|
|
15c97d5cfc | ||
|
|
905cbd5a35 | ||
|
|
5c0e2e6185 | ||
|
|
4836d2dacb | ||
|
|
bc81ec1ab5 | ||
|
|
8a4d039a37 | ||
|
|
468639264a | ||
|
|
9b649b470f | ||
|
|
edd5cf2bf7 | ||
|
|
67c605e640 | ||
|
|
8d4b878656 | ||
|
|
bf532dd1d0 | ||
|
|
eb1a125e59 | ||
|
|
256659a72d | ||
|
|
dbf3aa7e7d | ||
|
|
a75ce181ff | ||
|
|
37ae6b4ebf | ||
|
|
6631a6a95b | ||
|
|
918f8578ad | ||
|
|
10fa9f68ae | ||
|
|
6b5c9ccb0f | ||
|
|
652e8f1a9d | ||
|
|
fe53ae733d | ||
|
|
690dab2ae8 | ||
|
|
530542163e | ||
|
|
cf7f4c75c0 | ||
|
|
c6b395cf4c | ||
|
|
e72edeed62 | ||
|
|
3f22d53f89 | ||
|
|
08c0fa62b1 | ||
|
|
4c79cca079 | ||
|
|
67096242a2 | ||
|
|
f66112375d | ||
|
|
c5a082e248 | ||
|
|
c2a1b0aa33 | ||
|
|
a89dcc96d8 | ||
|
|
b2a10eafd9 | ||
|
|
f027aff89d | ||
|
|
b4eaf0d1c4 | ||
|
|
9d2975faaa | ||
|
|
0c0fdf9d78 | ||
|
|
e17afa0ef3 | ||
|
|
3d3eb6aa36 | ||
|
|
e22132255e | ||
|
|
ef041526ec | ||
|
|
704311dd9a | ||
|
|
9617a6f20f | ||
|
|
bc17442961 | ||
|
|
574d6818c1 | ||
|
|
5c416055ee | ||
|
|
fa30458dc6 | ||
|
|
638af93225 | ||
|
|
2c6ca741c7 | ||
|
|
d7f2df2540 | ||
|
|
b85ade1dba | ||
|
|
1c61a0b34e | ||
|
|
46b9449969 | ||
|
|
cc1bbd1231 | ||
|
|
b29cba2911 | ||
|
|
13bde3aa2c | ||
|
|
35b4f5fc97 | ||
|
|
1b0482b7d4 | ||
|
|
c9ff6eeab6 | ||
|
|
5ac57b743d | ||
|
|
f7b219701f | ||
|
|
d009aa3f8c | ||
|
|
078d49f883 | ||
|
|
f1bf517ba5 | ||
|
|
6269ed6f6e | ||
|
|
43b8dedbb1 | ||
|
|
978be7bd49 | ||
|
|
1b7f3aef62 | ||
|
|
7d24ee4867 | ||
|
|
269a3475b0 | ||
|
|
18e2bc6afc | ||
|
|
c6ef54e115 | ||
|
|
441b2831fc | ||
|
|
3cf4d60eca | ||
|
|
d228c89a1c | ||
|
|
f3b592e81b | ||
|
|
a5ff659fb4 | ||
|
|
ab80e2f3b3 | ||
|
|
2ef082f87e | ||
|
|
a1b97e76a7 | ||
|
|
185a078b4d | ||
|
|
28a4c944ee | ||
|
|
23d3553d99 | ||
|
|
fb09e4c46b | ||
|
|
4feb6f5fe1 | ||
|
|
df1b9efbbb | ||
|
|
553924fa1d | ||
|
|
a90dbd568f | ||
|
|
ca7ca1ce8e | ||
|
|
b3ab239a1e | ||
|
|
6e212c791c | ||
|
|
c6067ef147 | ||
|
|
8333f0c34e | ||
|
|
75becd8bb4 | ||
|
|
667376301a | ||
|
|
128e6b68a3 | ||
|
|
a7c2844ab4 | ||
|
|
bd0b7d274b | ||
|
|
bc57fad52b | ||
|
|
b81ef53cb9 | ||
|
|
c182bf0a08 | ||
|
|
07802b4b13 |
@@ -3,4 +3,5 @@
|
||||
/**/*/target
|
||||
/**/*/build-dir
|
||||
/**/*/java-runtime
|
||||
.idea
|
||||
.idea
|
||||
.vripper
|
||||
@@ -1,5 +1,278 @@
|
||||
# Changelog
|
||||
|
||||
## [3.2.3] - 2021-01-13
|
||||
### Changed
|
||||
- Finished posts are not cleared from the UI
|
||||
|
||||
## [3.2.2] - 2021-01-13
|
||||
### Changed
|
||||
- Set total download to max value
|
||||
|
||||
## [3.2.1] - 2021-01-12
|
||||
### Added
|
||||
- Add settings for connection timeout
|
||||
- Add settings for maximum attempts
|
||||
### Changed
|
||||
- Enhance posts rename logic
|
||||
- Dependencies update
|
||||
- Switch reactive from rx-java to reactor
|
||||
|
||||
## [3.2.0] - 2020-11-16
|
||||
### Changed
|
||||
- Fix build issue
|
||||
|
||||
## [3.1.0] - 2020-11-16
|
||||
### Changed
|
||||
- UI revamp
|
||||
- Fix bugs
|
||||
|
||||
## [3.0.11] - 2020-10-31
|
||||
### Added
|
||||
- Support for vipr.im
|
||||
|
||||
## [3.0.10] - 2020-10-04
|
||||
### Changed
|
||||
- Fix bug on settings update
|
||||
|
||||
## [3.0.9] - 2020-09-27
|
||||
### Changed
|
||||
- Revert to appending post id to download folders
|
||||
- Fix a bug that lead to adding duplicate posts
|
||||
- Remove old unused settings
|
||||
|
||||
## [3.0.8] - 2020-08-22
|
||||
### Changed
|
||||
- Bug fix which breaks concurrent downloads
|
||||
|
||||
## [3.0.7] - 2020-08-22
|
||||
### Changed
|
||||
- Fix MacOs build
|
||||
|
||||
## [3.0.6] - 2020-08-18
|
||||
### Changed
|
||||
- Fix TurboImageHost
|
||||
|
||||
## [3.0.5] - 2020-08-17
|
||||
### Changed
|
||||
- Update packages
|
||||
- Host class refactoring
|
||||
|
||||
## [3.0.4] - 2020-08-16
|
||||
### Changed
|
||||
- Improve rename UI
|
||||
|
||||
## [3.0.3] - 2020-08-16
|
||||
### Changed
|
||||
- Fix bugs with electron app
|
||||
|
||||
## [3.0.2] - 2020-08-15
|
||||
### Changed
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.1] - 2020-08-03
|
||||
### Changed
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.0] - 2020-08-03
|
||||
### Changed
|
||||
- Major rewrites
|
||||
- Introducing a proper database for persistence
|
||||
|
||||
## [2.12.1] - 2020-05-22
|
||||
### Changed
|
||||
- Fix alternative name bug
|
||||
|
||||
## [2.12.0] - 2020-05-21
|
||||
### Changed
|
||||
- Use new user hash from the api
|
||||
|
||||
## [2.11.7] - 2020-05-08
|
||||
### Changed
|
||||
- Fix bug with paths creation
|
||||
- Move Open Link button
|
||||
- Rename rename all title
|
||||
|
||||
## [2.11.6] - 2020-05-04
|
||||
### Changed
|
||||
- PNG filenames are now correctly named
|
||||
- Fix issue with special characters in post titles
|
||||
|
||||
## [2.11.5] - 2020-05-03
|
||||
### Changed
|
||||
- Reduce rows buffer size
|
||||
|
||||
## [2.11.4] - 2020-05-03
|
||||
### Changed
|
||||
- Code cleanup
|
||||
- Dependencies update
|
||||
|
||||
## [2.11.3] - 2020-04-30
|
||||
### Changed
|
||||
- Source code restructure
|
||||
- UI fixes and enhancements
|
||||
### Added
|
||||
- Add menu entry to rename a gallery to first alternative title
|
||||
- Add menu on toolbar for massive rename to first alternative title
|
||||
|
||||
## [2.11.2] - 2020-04-26
|
||||
### Changed
|
||||
- Bug fixes
|
||||
|
||||
## [2.11.1] - 2020-04-25
|
||||
### Changed
|
||||
- Code source maintenance
|
||||
### Added
|
||||
- Added option to rename gallery
|
||||
|
||||
## [2.11.0] - 2020-03-29
|
||||
### Changed
|
||||
- Upgrade Electron to v8
|
||||
- Upgrade Angular to v9
|
||||
- App settings are now portable in a json file
|
||||
- General code source maintenance
|
||||
### Added
|
||||
- Added Support for pixroute.com
|
||||
- Added grab post title from thread option
|
||||
|
||||
## [2.10.8] - 2020-01-25
|
||||
### Changed
|
||||
- Fix app url
|
||||
|
||||
## [2.10.7] - 2020-01-25
|
||||
### Changed
|
||||
- Drop SockJs and switch to WebSocket
|
||||
- Autoreconnect front to back
|
||||
|
||||
## [2.10.6] - 2020-01-19
|
||||
### Changed
|
||||
- Limit cache size
|
||||
|
||||
## [2.10.5] - 2020-01-18
|
||||
### Changed
|
||||
- Bug fixes
|
||||
|
||||
## [2.10.4] - 2020-01-18
|
||||
### Changed
|
||||
- Zero value for total max download will disable the limit
|
||||
- Add system notification
|
||||
- Bug fixes
|
||||
|
||||
## [2.10.3] - 2020-01-16
|
||||
### Changed
|
||||
- Some bug fixes
|
||||
- Enhance download queue logic
|
||||
- Clear cache button
|
||||
- Fix the partial status
|
||||
- Sanitize URLs with spaces
|
||||
### Added
|
||||
- Support for Postimg host
|
||||
- Support for Imagevenue host
|
||||
|
||||
## [2.10.2] - 2019-12-29
|
||||
### Changed
|
||||
- Upgrade electron to v7
|
||||
- Bug fix
|
||||
|
||||
## [2.10.1] - 2019-12-27
|
||||
### Changed
|
||||
- Bug fix
|
||||
|
||||
## [2.10.0] - 2019-12-27
|
||||
### Changed
|
||||
- Fix spring dependency bug
|
||||
### Added
|
||||
- Add support for imgspice
|
||||
|
||||
## [2.9.0] - 2019-12-27
|
||||
### Changed
|
||||
- Fix build issue
|
||||
|
||||
## [2.9.0] - 2019-12-27
|
||||
### Changed
|
||||
- Decrease timeout for fast fail
|
||||
### Added
|
||||
- Allow many hosts to run at the same time
|
||||
- Add title to gallery
|
||||
- Add global concurrent downloads
|
||||
- Add counter badge in link collector
|
||||
|
||||
## [2.8.1] - 2019-12-24
|
||||
### Changed
|
||||
- Fix issue with path creation
|
||||
|
||||
## [2.8.0] - 2019-12-24
|
||||
### Added
|
||||
- Photo gallery feature
|
||||
|
||||
## [2.7.1] - 2019-12-21
|
||||
### Added
|
||||
- Add version to title bar
|
||||
|
||||
## [2.7.0] - 2019-12-21
|
||||
### Changed
|
||||
- Add missing icon
|
||||
### Added
|
||||
- Add support for pixxxels
|
||||
|
||||
## [2.6.0] - 2019-12-21
|
||||
### Changed
|
||||
- Disable download button when nothing selected in Link collector grab menu
|
||||
- Better quality app icon
|
||||
- Implement context menu on download tab
|
||||
- Add stop remove and start button to context menu
|
||||
- Link will not be removed automatically from link collector tab
|
||||
- Some UI fixes
|
||||
|
||||
## [2.5.1] - 2019-12-09
|
||||
### Changed
|
||||
- Disable download button when nothing selected in Link collector grab menu
|
||||
### Added
|
||||
- Add hosts column in Link collector grab menu
|
||||
|
||||
## [2.5.0] - 2019-12-08
|
||||
### Changed
|
||||
- Change loading screen
|
||||
- UI Fixes
|
||||
- Fix bug with previews
|
||||
### Added
|
||||
- Add link collector feature
|
||||
- Add support for multiple links at a time
|
||||
|
||||
## [2.4.1] - 2019-11-26
|
||||
### Changed
|
||||
- Increase attempts for establishing ws connection between front and back end
|
||||
- Change default download folder to Home folder (installer), App folder (portable)
|
||||
|
||||
## [2.4.0] - 2019-11-23
|
||||
### Changed
|
||||
- Pimpandhost Host support
|
||||
|
||||
## [2.3.2] - 2019-11-23
|
||||
### Changed
|
||||
- Fix a bug with the scan component
|
||||
|
||||
## [2.3.1] - 2019-11-22
|
||||
### Changed
|
||||
- Fix connexion reset issue
|
||||
- Fix Post Status when autostart option is selected
|
||||
- Add stop all button
|
||||
|
||||
## [2.3.0] - 2019-11-21
|
||||
### Changed
|
||||
- UI enhancements and overhaul
|
||||
- Stability fixes with HTTP connections
|
||||
- MacOS official support
|
||||
- Use local sockjs and material icons font
|
||||
|
||||
## [2.2.5] - 2019-11-09
|
||||
### Changed
|
||||
- Exclude thumbs download for imagezilla and pixhost
|
||||
- Change retry on fail logic (may help avoid having download errors)
|
||||
|
||||
## [2.2.4] - 2019-10-05
|
||||
### Added
|
||||
- AppImage target for linux
|
||||
|
||||
## [2.2.3] - 2019-10-01
|
||||
### Changed
|
||||
- Strip libjvm.so for debian
|
||||
@@ -178,4 +451,4 @@
|
||||
- Add real time communication for progress updates via websocket (SockJS)
|
||||
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
|
||||
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
|
||||
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
|
||||
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
|
||||
|
||||
@@ -1,26 +1,22 @@
|
||||
# Viper Ripper!
|
||||
|
||||
This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
|
||||
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website.
|
||||
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
|
||||
|
||||
## How to build
|
||||
You need a recent version of maven 3.6.1+.
|
||||
|
||||
The application support 2 build modes:
|
||||
- Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
|
||||
- Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
|
||||
- Desktop app: Self contained app, built with electron includes Java runtimes.
|
||||
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
|
||||
|
||||
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
|
||||
|
||||
To build the server app:
|
||||
|
||||
mvn clean install
|
||||
mvn clean install -DskipTests
|
||||
To build the desktop app:
|
||||
|
||||
mvn clean install -Pelectron
|
||||
mvn clean install -Pelectron -DskipTests
|
||||
|
||||
Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
|
||||
|
||||
*Partially cross platform:
|
||||
Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
|
||||
The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>2.2.3</version>
|
||||
<version>3.2.3</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.1.1.RELEASE</version>
|
||||
<version>2.4.1</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
</parent>
|
||||
<modules>
|
||||
@@ -17,4 +17,4 @@
|
||||
<module>vripper-ui</module>
|
||||
<module>vripper-electron</module>
|
||||
</modules>
|
||||
</project>
|
||||
</project>
|
||||
|
||||
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 303 KiB |
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 4.0 KiB |
|
After Width: | Height: | Size: 592 B |
|
After Width: | Height: | Size: 8.3 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 1.5 KiB |
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 2.1 KiB |
@@ -1,13 +1,16 @@
|
||||
const { app, BrowserWindow } = require("electron");
|
||||
require('v8-compile-cache');
|
||||
const {app, BrowserWindow} = require("electron");
|
||||
const path = require("path");
|
||||
const url = require("url");
|
||||
const getPort = require("get-port");
|
||||
const { spawn } = require("child_process");
|
||||
const { ipcMain } = require("electron");
|
||||
const commandExists = require("command-exists").sync;
|
||||
const { dialog } = require("electron");
|
||||
const {spawn} = require("child_process");
|
||||
const {ipcMain} = require("electron");
|
||||
const {dialog} = require("electron");
|
||||
const axios = require('axios');
|
||||
const appDir = process.env.APPDIR;
|
||||
|
||||
// non null value when it is an AppImage
|
||||
const appImageDir = process.env.APPDIR;
|
||||
const appImagePath = process.env.APPIMAGE;
|
||||
|
||||
let win;
|
||||
let vripperServer;
|
||||
@@ -19,124 +22,150 @@ let terminationInteval;
|
||||
let terminated = false;
|
||||
|
||||
process.on("uncaughtException", err => {
|
||||
dialog.showErrorBox(err.message, err.stack);
|
||||
process.exit(1);
|
||||
dialog.showErrorBox(err.message, err.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
if (!commandExists("java")) {
|
||||
dialog.showErrorBox(
|
||||
"Java command is missing",
|
||||
"Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
|
||||
);
|
||||
process.exit(1);
|
||||
createWindow = () => {
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
app.setAppUserModelId("tn.mnlr.vripper");
|
||||
}
|
||||
|
||||
let icon;
|
||||
if (process.platform === "win32") {
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
icon = __dirname + '/icon.png';
|
||||
}
|
||||
win = new BrowserWindow({
|
||||
width: 1024,
|
||||
height: 800,
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
webPreferences: {
|
||||
nodeIntegration: true,
|
||||
enableRemoteModule: true
|
||||
},
|
||||
icon: icon
|
||||
});
|
||||
|
||||
win.removeMenu();
|
||||
win.setMenu(null);
|
||||
win.setMenuBarVisibility(false);
|
||||
|
||||
win.loadURL(
|
||||
url.format({
|
||||
pathname: path.join(__dirname, `vripper-ui/index.html`),
|
||||
protocol: "file:",
|
||||
slashes: true
|
||||
})
|
||||
);
|
||||
|
||||
// win.webContents.openDevTools();
|
||||
|
||||
win.on("closed", () => {
|
||||
win = null;
|
||||
});
|
||||
}
|
||||
|
||||
function createWindow() {
|
||||
let icon;
|
||||
if(process.platform === "win32") {
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
icon = __dirname + '/icon.png';
|
||||
}
|
||||
win = new BrowserWindow({
|
||||
width: 1024,
|
||||
height: 800,
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
webPreferences: {
|
||||
nodeIntegration: true
|
||||
},
|
||||
icon: icon
|
||||
});
|
||||
|
||||
win.removeMenu();
|
||||
win.setMenu(null);
|
||||
win.setMenuBarVisibility(false);
|
||||
|
||||
win.loadURL(
|
||||
url.format({
|
||||
pathname: path.join(__dirname, `vripper-ui/index.html`),
|
||||
protocol: "file:",
|
||||
slashes: true
|
||||
})
|
||||
);
|
||||
|
||||
// win.webContents.openDevTools();
|
||||
|
||||
win.on("closed", () => {
|
||||
win = null;
|
||||
});
|
||||
shutdownServer = () => {
|
||||
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
|
||||
headers: {'content-type': 'application/json'},
|
||||
}).then((response) => {
|
||||
terminationInteval = setInterval(() => {
|
||||
terminationAttemps++;
|
||||
if (terminated) {
|
||||
console.log('viper server terminated');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
} else if (terminationAttemps > maxTerminationAttemps) {
|
||||
console.log('viper server is not terminated');
|
||||
console.log('Proceed to kill');
|
||||
vripperServer.kill('SIGKILL');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
}
|
||||
}, 1000);
|
||||
}).catch((error) => {
|
||||
// Terminate immediately
|
||||
console.log(error);
|
||||
vripperServer.kill('SIGKILL');
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
}
|
||||
|
||||
getPort().then(port => {
|
||||
serverPort = port;
|
||||
ipcMain.on("get-port", event => {
|
||||
event.reply("port", port);
|
||||
});
|
||||
vripperServer = spawn(appDir !== undefined ? path.join(appDir, "java-runtime/bin/java") :
|
||||
path.join(app.getPath('exe'), "../java-runtime/bin/java"), [
|
||||
"-Xms256m",
|
||||
"-Xmx1024m",
|
||||
"-Dvripper.server.port=" + port,
|
||||
"-jar",
|
||||
appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
|
||||
path.join(app.getPath('exe'), "../bin/vripper-server.jar")
|
||||
], {
|
||||
stdio: 'ignore'
|
||||
});
|
||||
vripperServer.on('exit', (code, signal) => {
|
||||
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
});
|
||||
|
||||
const gotTheLock = app.requestSingleInstanceLock();
|
||||
|
||||
if (!gotTheLock) {
|
||||
app.quit();
|
||||
app.quit();
|
||||
} else {
|
||||
app.on("second-instance", (event, commandLine, workingDirectory) => {
|
||||
if (win) {
|
||||
if (win.isMinimized()) win.restore();
|
||||
win.focus();
|
||||
}
|
||||
});
|
||||
getPort().then(port => {
|
||||
serverPort = port;
|
||||
ipcMain.on("get-port", event => {
|
||||
event.reply("port", port);
|
||||
});
|
||||
|
||||
app.on("ready", createWindow);
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
if (process.platform !== "darwin") {
|
||||
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}).then((response) => {
|
||||
terminationInteval= setInterval(() => {
|
||||
terminationAttemps++;
|
||||
if(terminated) {
|
||||
console.log('viper server terminated');
|
||||
clearInterval(terminationInteval);
|
||||
const appPath = path.join(app.getAppPath(), '../../');
|
||||
let javaBinPath, jarPath, baseDir;
|
||||
if (appImageDir !== undefined && process.platform === 'linux') {
|
||||
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
|
||||
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
|
||||
baseDir = path.join(appImagePath, '..');
|
||||
} else if (process.platform === 'darwin') {
|
||||
javaBinPath = path.join(appPath, "java-runtime/bin/java");
|
||||
jarPath = path.join(appPath, "bin/vripper-server.jar");
|
||||
baseDir = app.getPath('appData');
|
||||
} else if (process.platform === 'win32') {
|
||||
javaBinPath = path.join(appPath, "java-runtime/bin/java");
|
||||
jarPath = path.join(appPath, "bin/vripper-server.jar");
|
||||
baseDir = appPath;
|
||||
} else {
|
||||
console.error(`Unknown platform ${process.platform}`);
|
||||
app.quit();
|
||||
} else if(terminationAttemps > maxTerminationAttemps) {
|
||||
console.log('viper server is not terminated');
|
||||
console.log('Proceed to kill');
|
||||
vripperServer.kill('SIGKILL');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
}
|
||||
}, 1000);
|
||||
})
|
||||
.catch((error) => {
|
||||
// Terminate immediately
|
||||
vripperServer.kill('SIGKILL');
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
app.on("activate", () => {
|
||||
if (win === null) {
|
||||
createWindow();
|
||||
}
|
||||
});
|
||||
vripperServer = spawn(javaBinPath, [
|
||||
"-Xms256m",
|
||||
"-Dvripper.server.port=" + port,
|
||||
"-Dbase.dir=" + baseDir,
|
||||
"-jar",
|
||||
jarPath
|
||||
], {
|
||||
stdio: 'ignore'
|
||||
});
|
||||
vripperServer.on('exit', (code, signal) => {
|
||||
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
});
|
||||
|
||||
app.on("second-instance", (event, commandLine, workingDirectory) => {
|
||||
if (win) {
|
||||
if (win.isMinimized()) win.restore();
|
||||
win.focus();
|
||||
}
|
||||
});
|
||||
|
||||
app.on("ready", createWindow);
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
if (process.platform !== "darwin") {
|
||||
shutdownServer();
|
||||
}
|
||||
});
|
||||
|
||||
app.on("will-quit", () => {
|
||||
if (process.platform === "darwin") {
|
||||
shutdownServer();
|
||||
}
|
||||
});
|
||||
|
||||
app.on("activate", () => {
|
||||
if (win === null) {
|
||||
createWindow();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-electron",
|
||||
"version": "2.2.3",
|
||||
"version": "3.2.3",
|
||||
"description": "A ripper for vipergirls.to built using web technolgies",
|
||||
"main": "main.js",
|
||||
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
|
||||
@@ -8,7 +8,7 @@
|
||||
"license": "ISC",
|
||||
"build": {
|
||||
"appId": "tn.mnlr.vripper",
|
||||
"productName": "vripper",
|
||||
"productName": "VRipper",
|
||||
"files": [
|
||||
"**/*",
|
||||
"!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
|
||||
@@ -38,34 +38,26 @@
|
||||
}
|
||||
],
|
||||
"win": {
|
||||
"icon": "icon.png",
|
||||
"icon": "icon.ico",
|
||||
"target": [
|
||||
"nsis"
|
||||
"dir"
|
||||
]
|
||||
},
|
||||
"nsis": {
|
||||
"createDesktopShortcut": "always",
|
||||
"oneClick": false,
|
||||
"perMachine": false
|
||||
},
|
||||
"linux": {
|
||||
"synopsis": "vipergirls.to ripper",
|
||||
"category": "Utility",
|
||||
"packageCategory": "Utility",
|
||||
"icon": "icons",
|
||||
"target": [
|
||||
"deb"
|
||||
"AppImage"
|
||||
]
|
||||
},
|
||||
"deb": {
|
||||
"depends": [
|
||||
"gconf2",
|
||||
"gconf-service",
|
||||
"libnotify4",
|
||||
"libappindicator1",
|
||||
"libxtst6",
|
||||
"libnss3"
|
||||
]
|
||||
"mac": {
|
||||
"category": "public.app-category.utilities",
|
||||
"target": [
|
||||
"dmg"
|
||||
],
|
||||
"icon": "icon.icns"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
@@ -73,17 +65,16 @@
|
||||
"dist": "node pre-build.js && electron-builder"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^6.0.9",
|
||||
"electron-builder": "^21.2.0"
|
||||
"electron": "11.1.1",
|
||||
"electron-builder": "22.9.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^0.19.0",
|
||||
"cheerio": "^1.0.0-rc.3",
|
||||
"command-exists": "^1.2.8",
|
||||
"copy-dir": "^1.1.0",
|
||||
"custom-electron-titlebar": "^3.0.10",
|
||||
"electron-context-menu": "^0.13.0",
|
||||
"get-port": "^5.0.0",
|
||||
"rimraf": "^2.6.3"
|
||||
"axios": "0.21.1",
|
||||
"cheerio": "1.0.0-rc.3",
|
||||
"copy-dir": "1.3.0",
|
||||
"electron-context-menu": "2.1.0",
|
||||
"get-port": "5.1.1",
|
||||
"rimraf": "3.0.2",
|
||||
"v8-compile-cache": "2.1.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,77 +1,77 @@
|
||||
<?xml version="1.0"?>
|
||||
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>2.2.3</version>
|
||||
</parent>
|
||||
<artifactId>vripper-electron</artifactId>
|
||||
<name>vripper-electron</name>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<configuration>
|
||||
<classifier>${buildClassifier}</classifier>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>electron</id>
|
||||
<properties>
|
||||
<buildClassifier>electron</buildClassifier>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>runtime</scope>
|
||||
<classifier>${buildClassifier}</classifier>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.2.3</version>
|
||||
</parent>
|
||||
<artifactId>vripper-electron</artifactId>
|
||||
<name>vripper-electron</name>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.github.eirslett</groupId>
|
||||
<artifactId>frontend-maven-plugin</artifactId>
|
||||
<version>1.3</version>
|
||||
<configuration>
|
||||
<nodeVersion>v12.2.0</nodeVersion>
|
||||
<npmVersion>6.9.0</npmVersion>
|
||||
<workingDirectory>build-dir</workingDirectory>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>install node and npm</id>
|
||||
<goals>
|
||||
<goal>install-node-and-npm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>npm install</id>
|
||||
<goals>
|
||||
<goal>npm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>npm run build</id>
|
||||
<goals>
|
||||
<goal>npm</goal>
|
||||
</goals>
|
||||
<plugin>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<configuration>
|
||||
<arguments>run dist</arguments>
|
||||
<classifier>${buildClassifier}</classifier>
|
||||
</configuration>
|
||||
<phase>generate-resources</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</build>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>electron</id>
|
||||
<properties>
|
||||
<buildClassifier>electron</buildClassifier>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>runtime</scope>
|
||||
<classifier>${buildClassifier}</classifier>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.github.eirslett</groupId>
|
||||
<artifactId>frontend-maven-plugin</artifactId>
|
||||
<version>1.3</version>
|
||||
<configuration>
|
||||
<nodeVersion>v12.2.0</nodeVersion>
|
||||
<npmVersion>6.9.0</npmVersion>
|
||||
<workingDirectory>build-dir</workingDirectory>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>install node and npm</id>
|
||||
<goals>
|
||||
<goal>install-node-and-npm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>npm install</id>
|
||||
<goals>
|
||||
<goal>npm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>npm run build</id>
|
||||
<goals>
|
||||
<goal>npm</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<arguments>run dist</arguments>
|
||||
</configuration>
|
||||
<phase>generate-resources</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
|
||||
const fs = require('fs');
|
||||
const rimraf = require("rimraf");
|
||||
const copydir = require("copy-dir");
|
||||
const { execSync } = require("child_process");
|
||||
const {execSync} = require("child_process");
|
||||
|
||||
rimraf.sync('./build/vripper-ui');
|
||||
copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
|
||||
@@ -19,7 +19,7 @@ fs.writeFileSync('./build/vripper-ui/index.html', $.html());
|
||||
console.log('Building runtime environment');
|
||||
rimraf.sync('java-runtime');
|
||||
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
|
||||
if(process.platform === 'linux') {
|
||||
if (process.platform === 'linux') {
|
||||
console.log('Stripping libjvm.so');
|
||||
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
|
||||
<output url="file://$MODULE_DIR$/target/classes" />
|
||||
<output-test url="file://$MODULE_DIR$/target/test-classes" />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<excludeFolder url="file://$MODULE_DIR$/target" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>2.2.3</version>
|
||||
<version>3.2.3</version>
|
||||
</parent>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<name>vripper-server</name>
|
||||
@@ -32,6 +32,20 @@
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-actuator</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.liquibase</groupId>
|
||||
<artifactId>liquibase-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hsqldb</groupId>
|
||||
<artifactId>hsqldb</artifactId>
|
||||
<version>2.5.1</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
@@ -58,15 +72,19 @@
|
||||
<version>2.22</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.reactivex.rxjava2</groupId>
|
||||
<artifactId>rxjava</artifactId>
|
||||
<version>2.2.6</version>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
<artifactId>reactor-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.jodah</groupId>
|
||||
<artifactId>failsafe</artifactId>
|
||||
<version>2.0.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>28.1-jre</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper-ui</artifactId>
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package tn.mnlr.vripper;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.event.ContextRefreshedEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tn.mnlr.vripper.jpa.repositories.IRepository;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
@Component
|
||||
public class EventListenerBean {
|
||||
|
||||
@Getter
|
||||
private static boolean init = false;
|
||||
|
||||
private final DataService dataService;
|
||||
private final Set<IRepository> repositorySet;
|
||||
|
||||
@Autowired
|
||||
public EventListenerBean(DataService dataService, Set<IRepository> repositorySet) {
|
||||
this.dataService = dataService;
|
||||
this.repositorySet = repositorySet;
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void onApplicationEvent(ContextRefreshedEvent event) {
|
||||
repositorySet.forEach(IRepository::init);
|
||||
dataService.setDownloadingToStopped();
|
||||
init = true;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package tn.mnlr.vripper;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
@@ -11,30 +10,24 @@ import org.springframework.stereotype.Component;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class SpringContext implements ApplicationContextAware {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SpringContext.class);
|
||||
|
||||
private static ConfigurableApplicationContext context;
|
||||
|
||||
/**
|
||||
* Returns the Spring managed bean instance of the given class type (if it exists).
|
||||
* Returns null otherwise.
|
||||
*
|
||||
* @param beanClass
|
||||
* @return
|
||||
*/
|
||||
public static <T extends Object> T getBean(Class<T> beanClass) {
|
||||
public static <T> T getBean(Class<T> beanClass) {
|
||||
return context.getBean(beanClass);
|
||||
}
|
||||
|
||||
public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
|
||||
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
|
||||
return context.getBeansOfType(beanClass);
|
||||
}
|
||||
|
||||
public static void close() {
|
||||
logger.info("Application terminating...");
|
||||
context.close();
|
||||
log.info("Application terminating...");
|
||||
if (context != null) {
|
||||
context.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -1,81 +1,20 @@
|
||||
package tn.mnlr.vripper;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tn.mnlr.vripper.exception.VripperException;
|
||||
import tn.mnlr.vripper.services.AppSettingsService;
|
||||
import tn.mnlr.vripper.services.PersistenceService;
|
||||
import tn.mnlr.vripper.services.VipergirlsAuthService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.io.File;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
@SpringBootApplication
|
||||
@Slf4j
|
||||
public class VripperApplication {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
|
||||
|
||||
// public static final String dataPath = System.getProperty("user.home", ".") + File.separator + ".vripper" + File.separator + "data.json";
|
||||
|
||||
public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
try {
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
|
||||
SpringApplication.run(VripperApplication.class, args);
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to run the application", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Component
|
||||
public class AppCommandRunner implements CommandLineRunner {
|
||||
|
||||
@Autowired
|
||||
private VipergirlsAuthService authService;
|
||||
|
||||
@Autowired
|
||||
private PersistenceService persistenceService;
|
||||
|
||||
@Autowired
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
@Value("${base.dir}")
|
||||
private String baseDir;
|
||||
|
||||
@Getter
|
||||
private String dataPath;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
|
||||
persistenceService.restore();
|
||||
appSettingsService.restore();
|
||||
registerShutdownHook();
|
||||
|
||||
try {
|
||||
authService.authenticate();
|
||||
} catch (VripperException e) {
|
||||
logger.error("Cannot authenticate user with ViperGirls", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void registerShutdownHook() {
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
|
||||
log.error("Failed to run the application", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.function.CheckedRunnable;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.impl.client.BasicCookieStore;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.stream.ImageInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.*;
|
||||
|
||||
@Slf4j
|
||||
public class DownloadJob implements CheckedRunnable {
|
||||
|
||||
private static final Object LOCK = new Object();
|
||||
private static final int READ_BUFFER_SIZE = 8192;
|
||||
private final DataService dataService;
|
||||
private final PathService pathService;
|
||||
private final ConnectionService cm;
|
||||
private final VGAuthService authService;
|
||||
private final DownloadSpeedService downloadSpeedService;
|
||||
private final SettingsService settingsService;
|
||||
private final HttpClientContext context;
|
||||
@Getter
|
||||
private final Image image;
|
||||
@Getter
|
||||
private final Post post;
|
||||
private boolean stopped = false;
|
||||
@Getter
|
||||
private boolean finished = false;
|
||||
|
||||
DownloadJob(Post post, Image image) {
|
||||
this.image = image;
|
||||
this.post = post;
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
pathService = SpringContext.getBean(PathService.class);
|
||||
cm = SpringContext.getBean(ConnectionService.class);
|
||||
authService = SpringContext.getBean(VGAuthService.class);
|
||||
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
|
||||
settingsService = SpringContext.getBean(SettingsService.class);
|
||||
context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
context.setAttribute(ContextAttributes.OPEN_CONNECTION.toString(), Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
|
||||
}
|
||||
|
||||
public void download(final Post post, final Image image) throws DownloadException {
|
||||
|
||||
try {
|
||||
|
||||
image.setStatus(Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
|
||||
synchronized (LOCK) {
|
||||
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
|
||||
post.setStatus(Status.DOWNLOADING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
}
|
||||
|
||||
// The post may be updated and the download directory might be set by another thread
|
||||
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
|
||||
if (updatedPost.getDownloadDirectory() == null) {
|
||||
pathService.createDefaultPostFolder(updatedPost);
|
||||
}
|
||||
if (settingsService.getSettings().getLeaveThanksOnStart()) {
|
||||
authService.leaveThanks(updatedPost);
|
||||
}
|
||||
}
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* HOST SPECIFIC
|
||||
*/
|
||||
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
|
||||
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
|
||||
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
|
||||
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
|
||||
/*
|
||||
* END HOST SPECIFIC
|
||||
*/
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
|
||||
log.debug(String.format("Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
|
||||
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
|
||||
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
|
||||
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
|
||||
httpGet.addHeader("Referer", image.getUrl());
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
|
||||
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
image.setTotal(response.getEntity().getContentLength());
|
||||
dataService.updateImageTotal(image.getTotal(), image.getId());
|
||||
|
||||
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
|
||||
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
|
||||
|
||||
byte[] buffer = new byte[READ_BUFFER_SIZE];
|
||||
int read;
|
||||
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
|
||||
fos.write(buffer, 0, read);
|
||||
image.increase(read);
|
||||
downloadSpeedService.increase(read);
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
}
|
||||
fos.flush();
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
checkImageTypeAndRename(dataService.findById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
throw new DownloadException(e);
|
||||
} finally {
|
||||
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
|
||||
image.setStatus(Status.COMPLETE);
|
||||
} else if (stopped) {
|
||||
image.setStatus(Status.STOPPED);
|
||||
} else {
|
||||
image.setStatus(Status.ERROR);
|
||||
}
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
|
||||
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
|
||||
if (!it.hasNext()) {
|
||||
throw new HostException("Image file is not recognized!");
|
||||
}
|
||||
ImageReader reader = it.next();
|
||||
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
|
||||
String imageNameLC = imageName.toLowerCase();
|
||||
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
|
||||
imageName += ".jpg";
|
||||
} else {
|
||||
String toReplace = null;
|
||||
if (imageNameLC.endsWith("_jpg")) {
|
||||
toReplace = "_jpg";
|
||||
} else if (imageNameLC.endsWith("_jpeg")) {
|
||||
toReplace = "_jpeg";
|
||||
}
|
||||
if (toReplace != null) {
|
||||
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
|
||||
}
|
||||
}
|
||||
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
|
||||
String imageNameLC = imageName.toLowerCase();
|
||||
if (!imageNameLC.endsWith("_png")) {
|
||||
imageName += ".png";
|
||||
} else {
|
||||
String toReplace = null;
|
||||
if (imageNameLC.endsWith("_png")) {
|
||||
toReplace = "_png";
|
||||
}
|
||||
if (toReplace != null) {
|
||||
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to guess image format", e);
|
||||
}
|
||||
try {
|
||||
pathService.getDirectoryAccess().lock();
|
||||
File downloadDestinationFolder = pathService.calcDownloadDirectory(post);
|
||||
File outImage = new File(downloadDestinationFolder, (settingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
|
||||
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to rename the image", e);
|
||||
} finally {
|
||||
pathService.getDirectoryAccess().unlock();
|
||||
try {
|
||||
Files.delete(outputFile.toPath());
|
||||
} catch (IOException e) {
|
||||
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
if (stopped) {
|
||||
finished = true;
|
||||
return;
|
||||
}
|
||||
log.debug(String.format("Starting downloading %s", image.getUrl()));
|
||||
download(post, image);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
DownloadJob that = (DownloadJob) o;
|
||||
return Objects.equals(image, that.image) &&
|
||||
Objects.equals(post, that.post);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(image, post);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
|
||||
if (requests != null) {
|
||||
for (AbstractExecutionAwareRequest request : requests) {
|
||||
request.abort();
|
||||
}
|
||||
}
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
public enum ContextAttributes {
|
||||
OPEN_CONNECTION("OPEN_CONNECTION");
|
||||
|
||||
private final String value;
|
||||
|
||||
ContextAttributes(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
@Slf4j
|
||||
public class DownloadRunnable implements Runnable {
|
||||
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final ConnectionService connectionService;
|
||||
|
||||
private final DownloadJob downloadJob;
|
||||
|
||||
public DownloadRunnable(final DownloadJob downloadJob) {
|
||||
downloadService = SpringContext.getBean(DownloadService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
connectionService = SpringContext.getBean(ConnectionService.class);
|
||||
this.downloadJob = downloadJob;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
Failsafe.with(connectionService.getRetryPolicy())
|
||||
.onFailure(e -> {
|
||||
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
|
||||
downloadJob.getImage().setStatus(Status.ERROR);
|
||||
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
|
||||
})
|
||||
.onComplete(e -> {
|
||||
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
|
||||
downloadService.afterJobFinish(downloadJob);
|
||||
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
|
||||
}).run(downloadJob);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.NonNull;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DownloadService {
|
||||
|
||||
private static final List<Status> FINISHED = Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
|
||||
private final int MAX_POOL_SIZE = 12;
|
||||
|
||||
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
|
||||
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
|
||||
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final SettingsService settings;
|
||||
private final DataService dataService;
|
||||
private final PostService postService;
|
||||
|
||||
private boolean pauseQ = false;
|
||||
private Thread executionThread;
|
||||
private Thread pollThread;
|
||||
|
||||
@Autowired
|
||||
public DownloadService(PendingQueue pendingQueue, SettingsService settings, DataService dataService, PostService postService) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
this.settings = settings;
|
||||
this.dataService = dataService;
|
||||
this.postService = postService;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
executionThread = new Thread(this::start, "Executor thread");
|
||||
pollThread = new Thread(this::poll, "Polling thread");
|
||||
pollThread.start();
|
||||
executionThread.start();
|
||||
}
|
||||
|
||||
public void destroy() throws Exception {
|
||||
log.info("Shutting down ExecutionService");
|
||||
executionThread.interrupt();
|
||||
pollThread.interrupt();
|
||||
executor.shutdown();
|
||||
dataService.findAllPosts().forEach(p -> {
|
||||
log.debug(String.format("Stopping download jobs for %s", p));
|
||||
this.stopRunning(p.getPostId());
|
||||
});
|
||||
executor.awaitTermination(5, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private void stopRunning(@NonNull String postId) {
|
||||
List<DownloadJob> stopping = new ArrayList<>();
|
||||
Iterator<DownloadJob> iterator = executing.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob downloadJob = iterator.next();
|
||||
if (postId.equals(downloadJob.getPost().getPostId())) {
|
||||
downloadJob.stop();
|
||||
iterator.remove();
|
||||
stopping.add(downloadJob);
|
||||
}
|
||||
}
|
||||
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(DownloadJob::isFinished);
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void stopAll(List<String> posIds) {
|
||||
if (posIds != null) {
|
||||
posIds.forEach(this::stop);
|
||||
} else {
|
||||
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
|
||||
}
|
||||
}
|
||||
|
||||
public void restartAll(List<String> posIds) {
|
||||
if (posIds != null) {
|
||||
posIds.forEach(this::restart);
|
||||
} else {
|
||||
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
|
||||
}
|
||||
}
|
||||
|
||||
private void restart(@NonNull String postId) {
|
||||
if (isPending(postId)) {
|
||||
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
|
||||
return;
|
||||
}
|
||||
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
|
||||
if (images.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
post.setStatus(Status.PENDING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
for (Image image : images) {
|
||||
try {
|
||||
pendingQueue.put(post, image);
|
||||
} catch (InterruptedException e) {
|
||||
log.warn("Thread was interrupted", e);
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isPending(String postId) {
|
||||
return pendingQueue.isPending(postId);
|
||||
}
|
||||
|
||||
private void stop(String postId) {
|
||||
try {
|
||||
pauseQ = true;
|
||||
final Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (post == null) {
|
||||
return;
|
||||
}
|
||||
if (FINISHED.contains(post.getStatus())) {
|
||||
return;
|
||||
}
|
||||
pendingQueue.stop(post);
|
||||
stopRunning(postId);
|
||||
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
|
||||
dataService.finishPost(post);
|
||||
postService.stopFetchingMetadata(post);
|
||||
} finally {
|
||||
pauseQ = false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean canRun(Host host) {
|
||||
boolean canRun;
|
||||
AtomicInteger count = threadCount.get(host);
|
||||
if (count == null) {
|
||||
threadCount.put(host, new AtomicInteger(0));
|
||||
}
|
||||
canRun = threadCount.get(host).get() < settings.getSettings().getMaxThreads() && (settings.getSettings().getMaxTotalThreads() == 0 ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE : threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getSettings().getMaxTotalThreads());
|
||||
if (canRun && !pauseQ) {
|
||||
threadCount.get(host).incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void poll() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
List<DownloadJob> peek = pendingQueue.peek();
|
||||
for (DownloadJob downloadJob : peek) {
|
||||
if (canRun(downloadJob.getImage().getHost())) {
|
||||
executionQueue.offer(downloadJob);
|
||||
pendingQueue.remove(downloadJob);
|
||||
}
|
||||
}
|
||||
synchronized (threadCount) {
|
||||
threadCount.wait(2_000);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
push(executionQueue.take());
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
log.error("Execution Service failed", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void push(DownloadJob downloadJob) {
|
||||
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
|
||||
executor.execute(new DownloadRunnable(downloadJob));
|
||||
executing.add(downloadJob);
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(DownloadJob downloadJob) {
|
||||
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
|
||||
if (count == 0) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
executing.remove(downloadJob);
|
||||
synchronized (threadCount) {
|
||||
threadCount.notify();
|
||||
}
|
||||
}
|
||||
|
||||
public int runningCount() {
|
||||
return executing.size();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.BlockingDeque;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PendingQueue {
|
||||
|
||||
private final DataService dataService;
|
||||
private final SettingsService settingsService;
|
||||
private final List<Host> hosts;
|
||||
|
||||
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
|
||||
|
||||
@Autowired
|
||||
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
|
||||
this.dataService = dataService;
|
||||
this.settingsService = settingsService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
|
||||
}
|
||||
|
||||
public void put(Post post, Image image) throws InterruptedException {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob = new DownloadJob(post, image);
|
||||
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
|
||||
checkKey(post.getPostId());
|
||||
toBeExecuted.get(post.getPostId()).incrementAndGet();
|
||||
}
|
||||
|
||||
private synchronized void checkKey(String postId) {
|
||||
if (!toBeExecuted.containsKey(postId)) {
|
||||
toBeExecuted.put(postId, new AtomicInteger(0));
|
||||
}
|
||||
}
|
||||
|
||||
public void remove(final DownloadJob downloadJob) {
|
||||
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
|
||||
}
|
||||
|
||||
public List<DownloadJob> peek() {
|
||||
List<DownloadJob> downloadJobs = new ArrayList<>();
|
||||
if (hosts.size() == 0) {
|
||||
return downloadJobs;
|
||||
}
|
||||
for (Host host : hosts) {
|
||||
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
|
||||
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
|
||||
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
|
||||
if (downloadJob != null) {
|
||||
downloadJobs.add(downloadJob);
|
||||
}
|
||||
}
|
||||
}
|
||||
return downloadJobs;
|
||||
}
|
||||
|
||||
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
|
||||
for (Image image : images) {
|
||||
put(post, image);
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
}
|
||||
|
||||
public void stop(Post post) {
|
||||
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
|
||||
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
|
||||
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
|
||||
entry.getValue().removeIf(predicate);
|
||||
decrement(post.getPostId());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isPending(String postId) {
|
||||
return toBeExecuted.containsKey(postId);
|
||||
}
|
||||
|
||||
public synchronized int decrement(String postId) {
|
||||
AtomicInteger counter = toBeExecuted.get(postId);
|
||||
if (counter == null) {
|
||||
return 0;
|
||||
}
|
||||
int count = counter.decrementAndGet();
|
||||
if (count == 0) {
|
||||
toBeExecuted.remove(postId);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package tn.mnlr.vripper.entities;
|
||||
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.processors.BehaviorProcessor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class Image {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(Image.class);
|
||||
|
||||
private AppStateService appStateService;
|
||||
|
||||
private Disposable subscription;
|
||||
|
||||
private final String type = "img";
|
||||
|
||||
private String postId;
|
||||
|
||||
private String postName;
|
||||
|
||||
private Host host;
|
||||
|
||||
private String url;
|
||||
|
||||
private int index;
|
||||
|
||||
private AtomicLong current = new AtomicLong(0);
|
||||
private Status status;
|
||||
private BehaviorProcessor<Image> imageStateProcessor;
|
||||
|
||||
@Setter
|
||||
private long total = 0;
|
||||
|
||||
public Image(String url, String postId, String postName, Host host, int index) {
|
||||
this.url = url;
|
||||
this.postId = postId;
|
||||
this.postName = postName;
|
||||
this.host = host;
|
||||
this.index = index;
|
||||
status = Status.PENDING;
|
||||
imageStateProcessor = BehaviorProcessor.create();
|
||||
}
|
||||
|
||||
public void setAppStateService(AppStateService appStateService) {
|
||||
this.appStateService = appStateService;
|
||||
appStateService.getCurrentImages().put(this.url, this);
|
||||
appStateService.getLiveImageUpdates().onNext(this);
|
||||
}
|
||||
|
||||
public void setStatus(Status status) {
|
||||
this.status = status;
|
||||
update();
|
||||
}
|
||||
|
||||
public boolean isCompleted() {
|
||||
return status.equals(Status.COMPLETE);
|
||||
}
|
||||
|
||||
public void init() {
|
||||
if (appStateService == null) {
|
||||
logger.warn("Attempting to init the image whilst the App State is null, unexpected behaviour could occur from this");
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
imageStateProcessor = BehaviorProcessor.create();
|
||||
subscription = imageStateProcessor
|
||||
.onBackpressureLatest()
|
||||
.doOnNext(appStateService::onImageUpdate)
|
||||
.subscribe();
|
||||
|
||||
current.set(0);
|
||||
status = Status.PENDING;
|
||||
imageStateProcessor.onNext(this);
|
||||
}
|
||||
|
||||
public void cleanup() {
|
||||
if (imageStateProcessor != null) {
|
||||
imageStateProcessor.onComplete();
|
||||
}
|
||||
if (subscription != null) {
|
||||
subscription.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public void setCurrent(int current) {
|
||||
this.current.set(current);
|
||||
update();
|
||||
}
|
||||
|
||||
public void increase(int read) {
|
||||
current.addAndGet(read);
|
||||
update();
|
||||
}
|
||||
|
||||
private void update() {
|
||||
if(imageStateProcessor == null) {
|
||||
return;
|
||||
}
|
||||
imageStateProcessor.onNext(this);
|
||||
if (isCompleted()) {
|
||||
imageStateProcessor.onComplete();
|
||||
subscription.dispose();
|
||||
imageStateProcessor = null;
|
||||
}
|
||||
}
|
||||
|
||||
public enum Status {
|
||||
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Image image = (Image) o;
|
||||
return url.equals(image.url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(url);
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package tn.mnlr.vripper.entities;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.ToString;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Getter
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class Post {
|
||||
|
||||
private AppStateService appStateService;
|
||||
|
||||
private Status status;
|
||||
|
||||
private final String type = "post";
|
||||
|
||||
private String postId;
|
||||
|
||||
private String threadTitle;
|
||||
|
||||
private String threadId;
|
||||
|
||||
private String title;
|
||||
|
||||
private String url;
|
||||
|
||||
private List<Image> images;
|
||||
|
||||
private Map<String, Object> metadata;
|
||||
|
||||
private AtomicInteger done = new AtomicInteger(0);
|
||||
|
||||
private int total;
|
||||
|
||||
private Set<String> hosts;
|
||||
|
||||
private boolean removed = false;
|
||||
|
||||
private String forum;
|
||||
|
||||
public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) {
|
||||
this.title = title;
|
||||
this.url = url;
|
||||
this.images = images;
|
||||
this.metadata = metadata;
|
||||
this.postId = postId;
|
||||
this.forum = forum;
|
||||
this.threadId = threadId;
|
||||
this.threadTitle = threadTitle;
|
||||
if (this.images.contains(null)) {
|
||||
System.out.println("Oops");
|
||||
}
|
||||
this.images.stream().map(e -> {
|
||||
if (e == null) {
|
||||
System.out.println("Am the cause");
|
||||
}
|
||||
return e.getHost();
|
||||
}).collect(Collectors.toSet());
|
||||
this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
|
||||
this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
|
||||
total = images.size();
|
||||
status = Status.PENDING;
|
||||
|
||||
}
|
||||
|
||||
public void setAppStateService(AppStateService appStateService) {
|
||||
this.appStateService = appStateService;
|
||||
this.appStateService.getCurrentPosts().put(postId, this);
|
||||
this.appStateService.getLivePostsState().onNext(this);
|
||||
}
|
||||
|
||||
public void setRemoved(boolean removed) {
|
||||
this.removed = removed;
|
||||
updateNotification();
|
||||
}
|
||||
|
||||
public void increase() {
|
||||
done.incrementAndGet();
|
||||
updateNotification();
|
||||
}
|
||||
|
||||
public void setStatus(Status status) {
|
||||
this.status = status;
|
||||
updateNotification();
|
||||
}
|
||||
|
||||
private void updateNotification() {
|
||||
if (appStateService != null) {
|
||||
appStateService.getLivePostsState().onNext(this);
|
||||
}
|
||||
}
|
||||
|
||||
public enum Status {
|
||||
PENDING, DOWNLOADING, COMPLETE, ERROR, PARTIAL, STOPPED
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Post post = (Post) o;
|
||||
return postId.equals(post.postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(postId);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package tn.mnlr.vripper.entities.mixin.persistance;
|
||||
|
||||
import com.fasterxml.jackson.databind.util.StdConverter;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
|
||||
public class HostToString extends StdConverter<Host, String> {
|
||||
|
||||
@Override
|
||||
public String convert(Host value) {
|
||||
return value.getClass().getSimpleName();
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.entities.mixin.persistance;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.processors.BehaviorProcessor;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
public abstract class ImagePersistanceMixin {
|
||||
|
||||
@JsonSerialize(converter = HostToString.class)
|
||||
@JsonDeserialize(converter = StringToHost.class)
|
||||
private Host host;
|
||||
|
||||
@JsonIgnore
|
||||
private BehaviorProcessor<Image> imageStateProcessor;
|
||||
|
||||
@JsonIgnore
|
||||
private Disposable subscription;
|
||||
|
||||
@JsonIgnore
|
||||
private String type;
|
||||
|
||||
@JsonIgnore
|
||||
public abstract boolean isCompleted();
|
||||
|
||||
@JsonIgnore
|
||||
private AppStateService appStateService;
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package tn.mnlr.vripper.entities.mixin.persistance;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
public abstract class PostPersistanceMixin {
|
||||
|
||||
@JsonIgnore
|
||||
private String type;
|
||||
|
||||
@JsonIgnore
|
||||
private AppStateService appStateService;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
private boolean removed;
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package tn.mnlr.vripper.entities.mixin.persistance;
|
||||
|
||||
import com.fasterxml.jackson.databind.util.StdConverter;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
public class StringToHost extends StdConverter<String, Host> {
|
||||
|
||||
private Collection<Host> hosts = SpringContext.getBeansOfType(Host.class).values();
|
||||
|
||||
@Override
|
||||
public Host convert(String value) {
|
||||
return hosts.stream().filter(e -> e.getClass().getSimpleName().equals(value)).findAny().orElse(null);
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package tn.mnlr.vripper.entities.mixin.ui;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.processors.BehaviorProcessor;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
public abstract class ImageUIMixin {
|
||||
|
||||
@JsonIgnore
|
||||
private Host host;
|
||||
|
||||
@JsonIgnore
|
||||
private BehaviorProcessor<Image> imageStateProcessor;
|
||||
|
||||
@JsonIgnore
|
||||
private Disposable subscription;
|
||||
|
||||
@JsonIgnore
|
||||
private AppStateService appStateService;
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
package tn.mnlr.vripper.entities.mixin.ui;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public abstract class PostUIMixin {
|
||||
|
||||
@JsonIgnore
|
||||
private List<Image> images;
|
||||
|
||||
@JsonIgnore
|
||||
private AppStateService appStateService;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class ImageUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public ImageUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class MetadataUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
private final Long postIdRef;
|
||||
|
||||
public MetadataUpdateEvent(Object source, Long id, Long postIdRef) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
this.postIdRef = postIdRef;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class PostRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final String postId;
|
||||
|
||||
public PostRemoveEvent(Object source, String postId) {
|
||||
super(source);
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class PostUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public PostUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class QueuedRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final String threadId;
|
||||
|
||||
public QueuedRemoveEvent(Object source, String threadId) {
|
||||
super(source);
|
||||
this.threadId = threadId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class QueuedUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public QueuedUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DownloadException(Exception e) {
|
||||
public DownloadException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class HostException extends Exception {
|
||||
public HostException(Exception e) {
|
||||
public HostException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ public class HostException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public HostException(String message, Exception e) {
|
||||
super(message,e);
|
||||
public HostException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class HtmlProcessorException extends Exception {
|
||||
|
||||
public HtmlProcessorException(Exception e) {
|
||||
public HtmlProcessorException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public PostParseException(String message, Exception e) {
|
||||
public PostParseException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
|
||||
public PostParseException(Exception e) {
|
||||
public PostParseException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class QueueException extends Exception {
|
||||
|
||||
public QueueException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public QueueException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
|
||||
public QueueException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class RenameException extends Exception {
|
||||
|
||||
public RenameException(String message, IOException e) {
|
||||
super(message, e);
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ public class VripperException extends Exception {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public VripperException(Exception e) {
|
||||
public VripperException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class XpathException extends Exception {
|
||||
public XpathException(Exception e) {
|
||||
public XpathException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,37 +1,49 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.HtmlProcessorService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class AcidimgHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(AcidimgHost.class);
|
||||
|
||||
private static final String host = "acidimg.cc";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
|
||||
public static final String IMG_XPATH = "//img[@class='centred']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
|
||||
private final ConnectionService cm;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public AcidimgHost(ConnectionService cm, HostService hostService, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
|
||||
this.cm = cm;
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -39,22 +51,27 @@ public class AcidimgHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
logger.info(String.format("Click button found for %s", url));
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url);
|
||||
HttpPost httpPost = cm.buildHttpPost(url, context);
|
||||
httpPost.addHeader("Referer", url);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
|
||||
@@ -64,9 +81,9 @@ public class AcidimgHost extends Host {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
logger.info(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
|
||||
logger.info(String.format("Cleaning response for %s", httpPost));
|
||||
log.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
log.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
@@ -76,23 +93,22 @@ public class AcidimgHost extends Host {
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if(imgNode == null) {
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Cannot find the image node");
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle.isEmpty() ? getDefaultImageName(imgUrl) : imgTitle);
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
|
||||
@@ -1,231 +1,38 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.stream.ImageInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Iterator;
|
||||
import java.util.Random;
|
||||
import java.util.Objects;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
abstract public class Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(Host.class);
|
||||
|
||||
private static final int READ_BUFFER_SIZE = 8192;
|
||||
|
||||
@Autowired
|
||||
protected HtmlProcessorService htmlProcessorService;
|
||||
|
||||
@Autowired
|
||||
protected XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
|
||||
@Autowired
|
||||
private DownloadSpeedService downloadSpeedService;
|
||||
|
||||
@Autowired
|
||||
private PathService pathService;
|
||||
|
||||
abstract public String getHost();
|
||||
|
||||
abstract public String getLookup();
|
||||
|
||||
public boolean isSupported(String url) {
|
||||
return url.contains(getHost());
|
||||
return url.contains(getLookup());
|
||||
}
|
||||
|
||||
public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
|
||||
public abstract HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException;
|
||||
|
||||
try {
|
||||
|
||||
appStateService.postDownloadingUpdate(image.getPostId());
|
||||
|
||||
imageFileData.setPageUrl(image.getUrl());
|
||||
|
||||
/**
|
||||
* HOST SPECIFIC
|
||||
*/
|
||||
logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
|
||||
setNameAndUrl(image.getUrl(), imageFileData);
|
||||
logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
|
||||
logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
|
||||
|
||||
logger.info(String.format("Building image request for %s", image.getUrl()));
|
||||
setImageRequest(imageFileData);
|
||||
/**
|
||||
* END HOST SPECIFIC
|
||||
*/
|
||||
|
||||
String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
|
||||
logger.info(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
|
||||
imageFileData.setImageName(formatImageFileName);
|
||||
File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
|
||||
logger.info(String.format("Saving to %s", destinationFolder.getPath()));
|
||||
if (!destinationFolder.exists()) {
|
||||
logger.info(String.format("Creating %s", destinationFolder.getPath()));
|
||||
destinationFolder.mkdirs();
|
||||
}
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
|
||||
logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
|
||||
|
||||
if(response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
|
||||
File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
|
||||
InputStream downloadStream = response.getEntity().getContent();
|
||||
FileOutputStream fos = new FileOutputStream(outputFile);
|
||||
try {
|
||||
image.setTotal(response.getEntity().getContentLength());
|
||||
logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
|
||||
logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
|
||||
|
||||
byte[] buffer = new byte[READ_BUFFER_SIZE];
|
||||
int read;
|
||||
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
|
||||
fos.write(buffer, 0, read);
|
||||
image.increase(read);
|
||||
downloadSpeedService.increase(read);
|
||||
}
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} finally {
|
||||
if (downloadStream != null) {
|
||||
downloadStream.close();
|
||||
}
|
||||
if (fos != null) {
|
||||
fos.close();
|
||||
}
|
||||
}
|
||||
checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if(Thread.interrupted()) {
|
||||
throw new InterruptedException("Download was interrupted");
|
||||
}
|
||||
throw new DownloadException(e);
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Host host = (Host) o;
|
||||
return Objects.equals(getHost(), host.getHost());
|
||||
}
|
||||
|
||||
private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
|
||||
String formatName;
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
|
||||
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
|
||||
if (!it.hasNext()) {
|
||||
throw new HostException("Image file is not recognized!");
|
||||
}
|
||||
ImageReader reader = it.next();
|
||||
formatName = reader.getFormatName();
|
||||
if (formatName.toUpperCase().equals("JPEG")) {
|
||||
formatName = "jpg";
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to guess image format", e);
|
||||
}
|
||||
try {
|
||||
File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
|
||||
if (outImage.exists()) {
|
||||
outImage.delete();
|
||||
}
|
||||
Files.move(outputFile.toPath(), outImage.toPath());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to rename the image", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Just for testing, you may ignore
|
||||
* @throws Exception
|
||||
*/
|
||||
private void randomFail() throws Exception {
|
||||
Random random = new Random();
|
||||
int i = random.nextInt(100);
|
||||
if (i < 50) {
|
||||
throw new Exception("Purpose error");
|
||||
}
|
||||
}
|
||||
|
||||
protected final String getDefaultImageName(final String imgUrl) {
|
||||
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
|
||||
logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
|
||||
return imgUrl;
|
||||
}
|
||||
|
||||
protected final Response getResponse(final String url) throws HostException {
|
||||
String basePage;
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url);
|
||||
Header[] headers;
|
||||
logger.info(String.format("Requesting %s", url));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
headers = response.getAllHeaders();
|
||||
basePage = EntityUtils.toString(response.getEntity());
|
||||
logger.debug(String.format("%s response: %n%s", url, basePage));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Cleaning %s response", url));
|
||||
return new Response(htmlProcessorService.clean(basePage), headers);
|
||||
} catch (HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void setImageRequest(final ImageFileData imageFileData) {
|
||||
HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
|
||||
httpGet.addHeader("Referer", imageFileData.getPageUrl());
|
||||
imageFileData.setImageRequest(httpGet);
|
||||
}
|
||||
|
||||
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
|
||||
|
||||
@Getter
|
||||
public static class Response {
|
||||
protected Response(Document document, Header[] headers) {
|
||||
this.document = document;
|
||||
this.headers = headers;
|
||||
}
|
||||
private Document document;
|
||||
private Header[] headers;
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(getHost());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
@@ -14,24 +13,33 @@ import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.HtmlProcessorService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImageBamHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
|
||||
|
||||
private static final String host = "imagebam.com";
|
||||
public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
|
||||
public static final String IMG_XPATH = "//img[@class='image']";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
|
||||
private static final String IMG_XPATH = "//img[@class='image']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final ConnectionService cm;
|
||||
private final XpathService xpathService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public ImageBamHost(HostService hostService, ConnectionService cm, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
|
||||
this.hostService = hostService;
|
||||
this.cm = cm;
|
||||
this.xpathService = xpathService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -39,37 +47,34 @@ public class ImageBamHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Response response = getResponse(url);
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
HostService.Response response = hostService.getResponse(url, context);
|
||||
Document doc = response.getDocument();
|
||||
Header[] headers = response.getHeaders();
|
||||
String cookies = Arrays.asList(headers)
|
||||
.stream()
|
||||
.filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
|
||||
.map(e -> e.getValue().split(";")[0].trim())
|
||||
.collect(Collectors.joining("; "));
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
logger.info(String.format("Getting cookies to use for %s", url));
|
||||
logger.info(String.format("Click button found for %s", url));
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url);
|
||||
HttpGet httpGet = cm.buildHttpGet(url, context);
|
||||
httpGet.addHeader("Referer", url);
|
||||
httpGet.addHeader("Cookie", cookies);
|
||||
logger.info(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
|
||||
log.debug(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
String s = EntityUtils.toString(res.getEntity());
|
||||
logger.debug(String.format("%s response is:%n%s", httpGet, s));
|
||||
logger.debug(String.format("Cleaning response for %s", httpGet));
|
||||
log.debug(String.format("%s response is:%n%s", httpGet, s));
|
||||
log.debug(String.format("Cleaning response for %s", httpGet));
|
||||
doc = htmlProcessorService.clean(s);
|
||||
EntityUtils.consumeQuietly(res.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
@@ -79,20 +84,19 @@ public class ImageBamHost extends Host {
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
|
||||
} catch(Exception e) {
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,26 +1,33 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImageTwistHost extends Host {
|
||||
|
||||
public static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String host = "imagetwist.com";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public ImageTwistHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -28,25 +35,29 @@ public class ImageTwistHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle);
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImageVenueHost extends Host {
|
||||
|
||||
private static final String host = "imagevenue.com";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
|
||||
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImageVenueHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
|
||||
//Button detected. No need to actually click it, just make the call again.
|
||||
resp = hostService.getResponse(url, context);
|
||||
doc = resp.getDocument();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Failed to locate image");
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,32 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImageZillaHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
|
||||
|
||||
private static final String host = "imagezilla.net";
|
||||
public static final String IMG_XPATH = "//img[@id='photo']";
|
||||
private static final String lookup = "imagezilla.net/show";
|
||||
private static final String IMG_XPATH = "//img[@id='photo']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public ImageZillaHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -28,16 +34,21 @@ public class ImageZillaHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
|
||||
logger.info(String.format("Resolving name for %s", url));
|
||||
if(titleNode != null) {
|
||||
log.debug(String.format("Resolving name for %s", url));
|
||||
if (titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
title = null;
|
||||
@@ -46,13 +57,12 @@ public class ImageZillaHost extends Host {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if(title == null || title.isEmpty()) {
|
||||
title = getDefaultImageName(url);
|
||||
if (title == null || title.isEmpty()) {
|
||||
title = hostService.getDefaultImageName(url);
|
||||
}
|
||||
|
||||
try {
|
||||
imageFileData.setImageUrl(url.replace("show", "images"));
|
||||
imageFileData.setImageName(title);
|
||||
return new HostService.NameUrl(title, url.replace("show", "images"));
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImgSpiceHost extends Host {
|
||||
|
||||
private static final String host = "imgspice.com";
|
||||
private static final String IMG_XPATH = "//img[@id='imgpreview']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,32 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImgboxHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
|
||||
|
||||
private static final String host = "imgbox.com";
|
||||
public static final String IMG_XPATH = "//img[@id='img']";
|
||||
private static final String IMG_XPATH = "//img[@id='img']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public ImgboxHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -28,25 +34,29 @@ public class ImgboxHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle);
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
@@ -16,24 +16,35 @@ import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.HtmlProcessorService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImxHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
|
||||
|
||||
private static final String host = "imx.to";
|
||||
public static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
|
||||
public static final String IMG_XPATH = "//img[@class='centred']";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final ConnectionService cm;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public ImxHost(HostService hostService, XpathService xpathService, ConnectionService cm, HtmlProcessorService htmlProcessorService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
this.cm = cm;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -41,16 +52,21 @@ public class ImxHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Response resp = getResponse(url);
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node contDiv;
|
||||
String value = null;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
Node node = contDiv.getAttributes().getNamedItem("value");
|
||||
if (node != null) {
|
||||
@@ -60,46 +76,43 @@ public class ImxHost extends Host {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
if (value == null) {
|
||||
throw new HostException("Failed to obtain value attribute from continue input");
|
||||
}
|
||||
logger.info(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", value));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
logger.info(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
|
||||
logger.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
if (value == null) {
|
||||
throw new HostException("Failed to obtain value attribute from continue input");
|
||||
}
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url, context);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", value));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
log.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
log.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
|
||||
} catch(Exception e) {
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PimpandhostHost extends Host {
|
||||
|
||||
private static final String host = "pimpandhost.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PimpandhostHost(HostService hostService, XpathService xpathService) {
|
||||
super();
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url;
|
||||
try {
|
||||
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle,
|
||||
imgUrl
|
||||
);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PixRouteHost extends Host {
|
||||
|
||||
private static final String host = "pixroute.com";
|
||||
private static final String IMG_XPATH = "//img[@id='imgpreview']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PixRouteHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Failed to locate image");
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
|
||||
imgNode.getAttributes().getNamedItem("src").getTextContent().trim()
|
||||
);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,32 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PixhostHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
|
||||
|
||||
private static final String host = "pixhost.to";
|
||||
public static final String IMG_XPATH = "//img[@id='image']";
|
||||
private static final String lookup = "pixhost.to/show";
|
||||
private static final String IMG_XPATH = "//img[@id='image']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public PixhostHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -28,25 +34,29 @@ public class PixhostHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info(String.format("Resolving name and image url for %s", url));
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
imageFileData.setImageUrl(imgUrl);
|
||||
imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1));
|
||||
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PixxxelsHost extends Host {
|
||||
|
||||
private static final String host = "pixxxels.cc";
|
||||
private static final String IMG_XPATH = "//*[@id='download']";
|
||||
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PixxxelsHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode, titleNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = titleNode.getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PostImgHost extends Host {
|
||||
|
||||
private static final String host = "postimg.cc";
|
||||
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
|
||||
private static final String IMG_XPATH = "//a[@id='download']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PostImgHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node urlNode, titleNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> hostService.getDefaultImageName(url));
|
||||
|
||||
return new HostService.NameUrl(imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,32 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.q.ImageFileData;
|
||||
import tn.mnlr.vripper.services.ConnectionManager;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class TurboImageHost extends Host {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
|
||||
|
||||
private static final String host = "turboimagehost.com";
|
||||
public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
|
||||
public static final String IMG_XPATH = "//img[@id='uImage']";
|
||||
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
|
||||
private static final String IMG_XPATH = "//img[@id='imageid']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
private ConnectionManager cm;
|
||||
public TurboImageHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -29,16 +34,21 @@ public class TurboImageHost extends Host {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
Document doc = getResponse(url).getDocument();
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
logger.info(String.format("Resolving name for %s", url));
|
||||
if(titleNode != null) {
|
||||
log.debug(String.format("Resolving name for %s", url));
|
||||
if (titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
title = null;
|
||||
@@ -47,14 +57,13 @@ public class TurboImageHost extends Host {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if(title == null || title.isEmpty()) {
|
||||
title = getDefaultImageName(url);
|
||||
if (title == null || title.isEmpty()) {
|
||||
title = hostService.getDefaultImageName(url);
|
||||
}
|
||||
|
||||
try {
|
||||
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
|
||||
imageFileData.setImageName(title);
|
||||
return new HostService.NameUrl(title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ViprImHost extends Host {
|
||||
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String host = "vipr.im";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ViprImHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package tn.mnlr.vripper.jpa;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.services.ThreadPoolService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.io.File;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Component
|
||||
@EnableScheduling
|
||||
public class Management {
|
||||
|
||||
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||
private static final Pattern BACKUP_FILE_PATTERN = Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final String backupFolder;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final DownloadService downloadService;
|
||||
|
||||
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName, ThreadPoolService threadPoolService, DownloadService downloadService) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.downloadService = downloadService;
|
||||
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
|
||||
try {
|
||||
threadPoolService.destroy();
|
||||
downloadService.destroy();
|
||||
this.jdbcTemplate.execute("SHUTDOWN");
|
||||
} catch (Exception ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
@Scheduled(cron = "0 0 0 ? * *")
|
||||
private void backup() {
|
||||
|
||||
for (File file : Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[]{})) {
|
||||
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
|
||||
if (matcher.find()) {
|
||||
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
|
||||
if (localDate.isEqual(LocalDate.now())) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create a backup file
|
||||
File backupFile = new File(backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
|
||||
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class Image {
|
||||
|
||||
@JsonIgnore
|
||||
protected Long id;
|
||||
|
||||
@JsonIgnore
|
||||
private Host host;
|
||||
|
||||
private String url;
|
||||
|
||||
private int index;
|
||||
|
||||
private long current = 0;
|
||||
|
||||
private long total = 0;
|
||||
|
||||
private Status status;
|
||||
|
||||
private String postId;
|
||||
|
||||
@JsonIgnore
|
||||
private Long postIdRef;
|
||||
|
||||
public Image(String postId, String url, Host host, int index) {
|
||||
this.postId = postId;
|
||||
this.url = url;
|
||||
this.host = host;
|
||||
this.index = index;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
|
||||
|
||||
public void increase(int read) {
|
||||
current += read;
|
||||
}
|
||||
|
||||
public void init() {
|
||||
current = 0;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Image image = (Image) o;
|
||||
return Objects.equals(url, image.url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(url);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
public class Metadata {
|
||||
|
||||
@JsonIgnore
|
||||
private Long id;
|
||||
|
||||
private List<String> resolvedNames = Collections.emptyList();
|
||||
|
||||
private String postedBy;
|
||||
|
||||
private Long postIdRef;
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class Post {
|
||||
|
||||
@JsonIgnore
|
||||
private Long id;
|
||||
|
||||
private Status status;
|
||||
|
||||
private String postId;
|
||||
|
||||
private String threadTitle;
|
||||
|
||||
private String threadId;
|
||||
|
||||
private String title;
|
||||
|
||||
private String url;
|
||||
|
||||
private int done;
|
||||
|
||||
private int total;
|
||||
|
||||
private Set<String> hosts;
|
||||
|
||||
private String forum;
|
||||
|
||||
@JsonIgnore
|
||||
private String securityToken;
|
||||
|
||||
@JsonIgnore
|
||||
private String downloadDirectory;
|
||||
|
||||
private boolean thanked;
|
||||
|
||||
private Set<String> previews = Collections.emptySet();
|
||||
|
||||
private Metadata metadata;
|
||||
|
||||
public Post(String title, String url, String postId, String threadId, String threadTitle, String forum, String securityToken) {
|
||||
this.title = title;
|
||||
this.url = url;
|
||||
this.postId = postId;
|
||||
this.forum = forum;
|
||||
this.threadId = threadId;
|
||||
this.threadTitle = threadTitle;
|
||||
this.securityToken = securityToken;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Post post = (Post) o;
|
||||
return Objects.equals(postId, post.postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(postId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@ToString
|
||||
public class Queued {
|
||||
|
||||
@JsonIgnore
|
||||
private Long id;
|
||||
|
||||
private String link;
|
||||
|
||||
private String threadId;
|
||||
|
||||
private String postId;
|
||||
|
||||
private int total = 0;
|
||||
|
||||
private boolean loading = true;
|
||||
|
||||
public Queued(String link, String threadId, String postId) {
|
||||
this();
|
||||
this.link = link;
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
|
||||
public void done() {
|
||||
this.loading = false;
|
||||
}
|
||||
|
||||
public void increment() {
|
||||
this.total++;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Queued that = (Queued) o;
|
||||
return Objects.equals(threadId, that.threadId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(threadId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package tn.mnlr.vripper.jpa.domain.enums;
|
||||
|
||||
public enum Status {
|
||||
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED, PARTIAL
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IImageRepository extends IRepository {
|
||||
|
||||
Image save(Image image);
|
||||
|
||||
void deleteAllByPostId(String postId);
|
||||
|
||||
List<Image> findByPostId(String postId);
|
||||
|
||||
Integer countError();
|
||||
|
||||
List<Image> findByPostIdAndIsNotCompleted(String postId);
|
||||
|
||||
int stopByPostIdAndIsNotCompleted(String postId);
|
||||
|
||||
List<Image> findByPostIdAndIsError(String postId);
|
||||
|
||||
Optional<Image> findById(Long id);
|
||||
|
||||
int updateStatus(Status status, Long id);
|
||||
|
||||
int updateCurrent(long current, Long id);
|
||||
|
||||
int updateTotal(long total, Long id);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IMetadataRepository extends IRepository {
|
||||
|
||||
Metadata save(Metadata metadata);
|
||||
|
||||
Optional<Metadata> findById(Long id);
|
||||
|
||||
Optional<Metadata> findByPostId(String postId);
|
||||
|
||||
int deleteByPostId(String postId);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IPostRepository extends IRepository {
|
||||
|
||||
Post save(Post post);
|
||||
|
||||
Optional<Post> findByPostId(String postId);
|
||||
|
||||
Optional<Post> findById(Long id);
|
||||
|
||||
List<String> findCompleted();
|
||||
|
||||
List<Post> findAll();
|
||||
|
||||
boolean existByPostId(String postId);
|
||||
|
||||
int setDownloadingToStopped();
|
||||
|
||||
int deleteByPostId(String postId);
|
||||
|
||||
int updateStatus(Status status, Long id);
|
||||
|
||||
int updateDone(int done, Long id);
|
||||
|
||||
int updateFolderName(String postFolderName, Long id);
|
||||
|
||||
int updateTitle(String title, Long id);
|
||||
|
||||
int updateThanked(boolean thanked, Long id);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IQueuedRepository extends IRepository {
|
||||
Queued save(Queued queued);
|
||||
|
||||
Optional<Queued> findByThreadId(String threadId);
|
||||
|
||||
List<Queued> findAll();
|
||||
|
||||
Optional<Queued> findById(Long id);
|
||||
|
||||
int deleteByThreadId(String threadId);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
public interface IRepository {
|
||||
|
||||
void init();
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.ImageUpdateEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Service
|
||||
public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final AtomicLong counter = new AtomicLong(0);
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
@Autowired
|
||||
public ImageRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
Long maxId = jdbcTemplate.queryForObject(
|
||||
"SELECT MAX(ID) FROM IMAGE",
|
||||
Long.class
|
||||
);
|
||||
if (maxId == null) {
|
||||
maxId = 0L;
|
||||
}
|
||||
counter.set(maxId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Image save(Image image) {
|
||||
long id = counter.incrementAndGet();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
image.getCurrent(),
|
||||
image.getHost().getHost(),
|
||||
image.getIndex(),
|
||||
image.getPostId(),
|
||||
image.getStatus().name(),
|
||||
image.getTotal(),
|
||||
image.getUrl(),
|
||||
image.getPostIdRef()
|
||||
);
|
||||
image.setId(id);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return image;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAllByPostId(String postId) {
|
||||
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostId(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE WHERE POST_ID = ?",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countError() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'",
|
||||
Integer.class
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int stopByPostIdAndIsNotCompleted(String postId) {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
|
||||
postId
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostIdAndIsError(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Image> findById(Long id) {
|
||||
List<Image> images = jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.ID = ?",
|
||||
new ImageRowMapper(),
|
||||
id);
|
||||
if (images.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(images.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateStatus(Status status, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?",
|
||||
status.name(), id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateCurrent(long current, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?",
|
||||
current, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateTotal(long total, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?",
|
||||
total, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class ImageRowMapper implements RowMapper<Image> {
|
||||
|
||||
@Override
|
||||
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Image image = new Image();
|
||||
image.setId(rs.getLong("ID"));
|
||||
String host = rs.getString("HOST");
|
||||
image.setHost(SpringContext.getBeansOfType(Host.class).values().stream().filter(e -> e.getHost().equals(host)).findAny().orElse(null));
|
||||
image.setUrl(rs.getString("URL"));
|
||||
image.setIndex(rs.getInt("INDEX"));
|
||||
image.setCurrent(rs.getLong("CURRENT"));
|
||||
image.setTotal(rs.getLong("TOTAL"));
|
||||
image.setStatus(Status.valueOf(rs.getString("STATUS")));
|
||||
image.setPostId(rs.getString("POST_ID"));
|
||||
image.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.MetadataUpdateEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Service
|
||||
public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final AtomicLong counter = new AtomicLong(0);
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
@Autowired
|
||||
public MetadataRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
Long maxId = jdbcTemplate.queryForObject(
|
||||
"SELECT MAX(ID) FROM IMAGE",
|
||||
Long.class
|
||||
);
|
||||
if (maxId == null) {
|
||||
maxId = 0L;
|
||||
}
|
||||
counter.set(maxId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Metadata save(Metadata metadata) {
|
||||
long id = counter.incrementAndGet();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO METADATA (ID, POSTED_BY, RESOLVED_NAMES, POST_ID_REF) VALUES (?,?,?,?)",
|
||||
id,
|
||||
metadata.getPostedBy(),
|
||||
String.join("%sep%", metadata.getResolvedNames()),
|
||||
metadata.getPostIdRef()
|
||||
);
|
||||
metadata.setId(id);
|
||||
applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, id, metadata.getPostIdRef()));
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Metadata> findById(Long id) {
|
||||
List<Metadata> metadata = jdbcTemplate.query(
|
||||
"SELECT * FROM METADATA AS metadata WHERE metadata.ID = ?",
|
||||
new Object[]{id},
|
||||
new MetadataRowMapper()
|
||||
);
|
||||
if (metadata.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(metadata.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Metadata> findByPostId(String postId) {
|
||||
List<Metadata> metadata = jdbcTemplate.query(
|
||||
"SELECT metadata.* FROM METADATA AS metadata INNER JOIN POST post ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?",
|
||||
new Object[]{postId},
|
||||
new MetadataRowMapper()
|
||||
);
|
||||
if (metadata.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(metadata.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByPostId(String postId) {
|
||||
return jdbcTemplate.update(
|
||||
"DELETE FROM METADATA AS metadata WHERE metadata.ID = (SELECT inner_metadata.ID FROM POST AS post INNER JOIN METADATA inner_metadata ON post.ID = inner_metadata.POST_ID_REF WHERE post.POST_ID = ?)",
|
||||
postId
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
public class MetadataRowMapper implements RowMapper<Metadata> {
|
||||
|
||||
@Override
|
||||
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setId(rs.getLong("ID"));
|
||||
metadata.setPostedBy(rs.getString("POSTED_BY"));
|
||||
String resolvedNames = rs.getString("RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.PostRemoveEvent;
|
||||
import tn.mnlr.vripper.event.PostUpdateEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Service
|
||||
public class PostRepository implements IPostRepository, ApplicationEventPublisherAware {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final AtomicLong counter = new AtomicLong(0);
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
@Autowired
|
||||
public PostRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
Long maxId = jdbcTemplate.queryForObject(
|
||||
"SELECT MAX(ID) FROM POST",
|
||||
Long.class
|
||||
);
|
||||
if (maxId == null) {
|
||||
maxId = 0L;
|
||||
}
|
||||
counter.set(maxId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Post save(Post post) {
|
||||
long id = counter.incrementAndGet();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
post.getDone(),
|
||||
post.getForum(),
|
||||
String.join(";", post.getHosts()),
|
||||
post.getDownloadDirectory(),
|
||||
post.getPostId(),
|
||||
String.join(";", post.getPreviews()),
|
||||
post.getSecurityToken(),
|
||||
post.getStatus().name(),
|
||||
post.isThanked(),
|
||||
post.getThreadId(),
|
||||
post.getThreadTitle(),
|
||||
post.getTitle(),
|
||||
post.getTotal(),
|
||||
post.getUrl()
|
||||
);
|
||||
post.setId(id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return post;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Post> findByPostId(String postId) {
|
||||
List<Post> posts = jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE POST_ID = ?",
|
||||
new PostRowMapper(),
|
||||
postId);
|
||||
if (posts.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(posts.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> findCompleted() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT POST_ID FROM POST WHERE status = 'COMPLETE' AND done >= total",
|
||||
((rs, rowNum) -> rs.getString("POST_ID"))
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Post> findById(Long id) {
|
||||
List<Post> posts = jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
|
||||
new PostRowMapper(),
|
||||
id);
|
||||
if (posts.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(posts.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Post> findAll() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
|
||||
new PostRowMapper()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean existByPostId(String postId) {
|
||||
Integer count = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?",
|
||||
Integer.class,
|
||||
postId);
|
||||
if (count == null) {
|
||||
return false;
|
||||
} else {
|
||||
return count > 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDownloadingToStopped() {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'"
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByPostId(String postId) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"DELETE FROM POST AS post WHERE post.POST_ID = ?",
|
||||
postId
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostRemoveEvent(PostRepository.class, postId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateStatus(Status status, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?",
|
||||
status.name(), id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateDone(int done, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?",
|
||||
done, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateFolderName(String postFolderName, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
|
||||
postFolderName, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateTitle(String title, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?",
|
||||
title, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateThanked(boolean thanked, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?",
|
||||
thanked, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class PostRowMapper implements RowMapper<Post> {
|
||||
|
||||
private static final String DELIMITER = ";";
|
||||
|
||||
@Override
|
||||
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
|
||||
Post post = new Post();
|
||||
post.setId(rs.getLong("post.ID"));
|
||||
post.setStatus(Status.valueOf(rs.getString("STATUS")));
|
||||
post.setPostId(rs.getString("POST_ID"));
|
||||
post.setThreadTitle(rs.getString("THREAD_TITLE"));
|
||||
post.setThreadId(rs.getString("THREAD_ID"));
|
||||
post.setTitle(rs.getString("TITLE"));
|
||||
post.setUrl(rs.getString("URL"));
|
||||
post.setDone(rs.getInt("DONE"));
|
||||
post.setTotal(rs.getInt("TOTAL"));
|
||||
post.setHosts(Set.of(rs.getString("HOSTS").split(DELIMITER)));
|
||||
post.setForum(rs.getString("FORUM"));
|
||||
post.setSecurityToken(rs.getString("SECURITY_TOKEN"));
|
||||
post.setDownloadDirectory(rs.getString("POST_FOLDER_NAME"));
|
||||
post.setThanked(rs.getBoolean("THANKED"));
|
||||
String previews;
|
||||
if ((previews = rs.getString("PREVIEWS")) != null) {
|
||||
post.setPreviews(Set.of(previews.split(DELIMITER)));
|
||||
}
|
||||
|
||||
Long metadataId = rs.getLong("metadata.ID");
|
||||
if (!rs.wasNull()) {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setId(metadataId);
|
||||
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
String resolvedNames = rs.getString("RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
metadata.setPostedBy(rs.getString("POSTED_BY"));
|
||||
post.setMetadata(metadata);
|
||||
}
|
||||
|
||||
return post;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.QueuedRemoveEvent;
|
||||
import tn.mnlr.vripper.event.QueuedUpdateEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Service
|
||||
public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final AtomicLong counter = new AtomicLong(0);
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
@Autowired
|
||||
public QueuedRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
Long maxId = jdbcTemplate.queryForObject(
|
||||
"SELECT MAX(ID) FROM QUEUED",
|
||||
Long.class
|
||||
);
|
||||
if (maxId == null) {
|
||||
maxId = 0L;
|
||||
}
|
||||
counter.set(maxId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Queued save(Queued queued) {
|
||||
long id = counter.incrementAndGet();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
|
||||
id,
|
||||
queued.getTotal(),
|
||||
queued.getLink(),
|
||||
queued.isLoading(),
|
||||
queued.getPostId(),
|
||||
queued.getThreadId()
|
||||
);
|
||||
queued.setId(id);
|
||||
applicationEventPublisher.publishEvent(new QueuedUpdateEvent(QueuedRepository.class, id));
|
||||
return queued;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Queued> findByThreadId(String threadId) {
|
||||
List<Queued> queuedList = jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
|
||||
new Object[]{threadId},
|
||||
new QueuedRowMapper()
|
||||
);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(queuedList.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Queued> findAll() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED",
|
||||
new QueuedRowMapper()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Queued> findById(Long id) {
|
||||
List<Queued> queuedList = jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?",
|
||||
new Object[]{id},
|
||||
new QueuedRowMapper()
|
||||
);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(queuedList.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByThreadId(String threadId) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?",
|
||||
threadId
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new QueuedRemoveEvent(QueuedRepository.class, threadId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class QueuedRowMapper implements RowMapper<Queued> {
|
||||
|
||||
@Override
|
||||
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Queued queued = new Queued();
|
||||
queued.setId(rs.getLong("ID"));
|
||||
queued.setLink(rs.getString("LINK"));
|
||||
queued.setThreadId(rs.getString("THREAD_ID"));
|
||||
queued.setPostId(rs.getString("POST_ID"));
|
||||
queued.setTotal(rs.getInt("TOTAL"));
|
||||
queued.setLoading(rs.getBoolean("LOADING"));
|
||||
return queued;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public interface DataEventListener<T> {
|
||||
Flux<T> getDataFlux();
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Sinks.EmitFailureHandler;
|
||||
|
||||
public class EmitHandler {
|
||||
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.ImageUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class ImageUpdateEventListener implements ApplicationListener<ImageUpdateEvent>, DataEventListener<ImageUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<ImageUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(ImageUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ImageUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.MetadataUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class MetadataUpdateEventListener implements ApplicationListener<MetadataUpdateEvent>, DataEventListener<MetadataUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<MetadataUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(MetadataUpdateEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<MetadataUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.PostRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class PostRemoveEventListener implements ApplicationListener<PostRemoveEvent>, DataEventListener<PostRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<PostRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(PostRemoveEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<PostRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.PostUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class PostUpdateEventListener implements ApplicationListener<PostUpdateEvent>, DataEventListener<PostUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<PostUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(PostUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<PostUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.QueuedRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueuedRemoveEventListener implements ApplicationListener<QueuedRemoveEvent>, DataEventListener<QueuedRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<QueuedRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(QueuedRemoveEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<QueuedRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.QueuedUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueuedUpdateEventListener implements ApplicationListener<QueuedUpdateEvent>, DataEventListener<QueuedUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<QueuedUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(QueuedUpdateEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<QueuedUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package tn.mnlr.vripper.q;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
public class DownloadJob implements Callable<Image> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DownloadJob.class);
|
||||
|
||||
@Getter
|
||||
private final Image image;
|
||||
|
||||
@Getter
|
||||
private final ImageFileData imageFileData = new ImageFileData();
|
||||
|
||||
public DownloadJob(Image image) {
|
||||
this.image = image;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Image call() throws Exception {
|
||||
|
||||
logger.info(String.format("Starting downloading %s", image.getUrl()));
|
||||
image.setStatus(Image.Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
image.getHost().download(image, imageFileData);
|
||||
image.setStatus(Image.Status.COMPLETE);
|
||||
return image;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
DownloadJob that = (DownloadJob) o;
|
||||
return image.equals(that.image);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(image);
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package tn.mnlr.vripper.q;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DownloadQ {
|
||||
|
||||
private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
|
||||
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
|
||||
@Autowired
|
||||
private ExecutionService executionService;
|
||||
|
||||
private BlockingQueue<DownloadJob> downloadQ = new LinkedBlockingQueue<>();
|
||||
|
||||
@Getter
|
||||
private boolean notPauseQ = true;
|
||||
|
||||
public void put(Image image) throws InterruptedException {
|
||||
synchronized (appStateService) {
|
||||
logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
DownloadJob downloadJob = new DownloadJob(image);
|
||||
downloadQ.put(downloadJob);
|
||||
appStateService.newDownloadJob(downloadJob);
|
||||
}
|
||||
}
|
||||
|
||||
public DownloadJob take() throws InterruptedException {
|
||||
DownloadJob downloadJob = downloadQ.take();
|
||||
logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
|
||||
return downloadJob;
|
||||
}
|
||||
|
||||
public synchronized void enqueue(Post post) throws InterruptedException {
|
||||
for (Image image : post.getImages()) {
|
||||
put(image);
|
||||
}
|
||||
}
|
||||
|
||||
public void restart(String postId) throws InterruptedException {
|
||||
synchronized (appStateService) {
|
||||
if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
|
||||
logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
|
||||
return;
|
||||
}
|
||||
List<Image> images = appStateService.getPost(postId)
|
||||
.getImages()
|
||||
.stream()
|
||||
.filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
|
||||
.collect(Collectors.toList());
|
||||
if (images.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
appStateService.getPost(postId).setStatus(Post.Status.PENDING);
|
||||
logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
for (Image image : images) {
|
||||
put(image);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void removeScheduled(Image image) {
|
||||
synchronized (appStateService) {
|
||||
image.setStatus(Image.Status.STOPPED);
|
||||
logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
|
||||
|
||||
Iterator<DownloadJob> iterator = downloadQ.iterator();
|
||||
boolean removed = false;
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob next = iterator.next();
|
||||
if (next.getImage().getPostId().equals(image.getPostId())) {
|
||||
iterator.remove();
|
||||
appStateService.doneDownloadJob(image);
|
||||
logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
|
||||
removed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!removed) {
|
||||
logger.warn(String.format("Job for %s does not exist", image.getUrl()));
|
||||
}
|
||||
|
||||
image.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
public void removeRunning(String postId) {
|
||||
logger.info(String.format("Interrupting running jobs for post id %s", postId));
|
||||
executionService.stop(postId);
|
||||
}
|
||||
|
||||
|
||||
public void stop(String postId) {
|
||||
try {
|
||||
synchronized (appStateService) {
|
||||
if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
|
||||
return;
|
||||
}
|
||||
notPauseQ = false;
|
||||
appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
|
||||
List<Image> images = appStateService.getPost(postId)
|
||||
.getImages()
|
||||
.stream()
|
||||
.filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
|
||||
.collect(Collectors.toList());
|
||||
if (images.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
|
||||
images.forEach(image -> removeScheduled(image));
|
||||
removeRunning(postId);
|
||||
}
|
||||
} finally {
|
||||
notPauseQ = true;
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return downloadQ.size();
|
||||
}
|
||||
|
||||
public void stopAll() {
|
||||
synchronized (appStateService) {
|
||||
appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
|
||||
}
|
||||
}
|
||||
|
||||
public void restartAll() throws InterruptedException {
|
||||
synchronized (appStateService) {
|
||||
for (Post post : appStateService.getCurrentPosts().values()) {
|
||||
String postId = post.getPostId();
|
||||
restart(postId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package tn.mnlr.vripper.q;
|
||||
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import net.jodah.failsafe.FailsafeException;
|
||||
import net.jodah.failsafe.RetryPolicy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.services.AppSettingsService;
|
||||
import tn.mnlr.vripper.services.AppStateService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class ExecutionService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
|
||||
|
||||
@Autowired
|
||||
private DownloadQ downloadQ;
|
||||
|
||||
@Autowired
|
||||
private AppSettingsService settings;
|
||||
|
||||
@Autowired
|
||||
private AppStateService appStateService;
|
||||
|
||||
private AtomicInteger threadCount = new AtomicInteger();
|
||||
|
||||
private ExecutorService executor = Executors.newFixedThreadPool(10);
|
||||
|
||||
private Thread executionThread;
|
||||
|
||||
private RetryPolicy<Object> retryPolicy;
|
||||
|
||||
private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
|
||||
retryPolicy = new RetryPolicy<>()
|
||||
.handleIf(e -> !(e instanceof InterruptedException))
|
||||
.withDelay(Duration.ofSeconds(5))
|
||||
.withMaxRetries(2)
|
||||
.abortOn(InterruptedException.class)
|
||||
.onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
|
||||
|
||||
executionThread = new Thread(this::start, "Executor thread");
|
||||
executionThread.start();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() throws InterruptedException {
|
||||
logger.info("Shutting down ExecutionService");
|
||||
executionThread.interrupt();
|
||||
executor.shutdown();
|
||||
appStateService.getCurrentPosts().keySet().forEach(p -> {
|
||||
logger.info(String.format("Stopping download jobs for %s", p));
|
||||
this.stop(p);
|
||||
});
|
||||
executor.awaitTermination(10, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
public void stop(String postId) {
|
||||
List<DownloadJob> data = running
|
||||
.stream()
|
||||
.filter(e -> e.getImage().getPostId().equals(postId))
|
||||
.peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
|
||||
.collect(Collectors.toList());
|
||||
logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
|
||||
|
||||
data.forEach(e -> {
|
||||
futures.get(e.getImage().getUrl()).cancel(true);
|
||||
if(e.getImageFileData().getImageRequest() != null) {
|
||||
e.getImageFileData().getImageRequest().abort();
|
||||
}
|
||||
e.getImage().cleanup();
|
||||
});
|
||||
}
|
||||
|
||||
boolean canRun() {
|
||||
boolean canRun = threadCount.get() < settings.getMaxThreads();
|
||||
if (canRun && downloadQ.isNotPauseQ()) {
|
||||
threadCount.incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
if (canRun()) {
|
||||
DownloadJob take = null;
|
||||
try {
|
||||
take = downloadQ.take();
|
||||
if (take == null) {
|
||||
continue;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
|
||||
DownloadJob finalTake = take;
|
||||
Runnable task = () -> {
|
||||
running.add(finalTake);
|
||||
|
||||
Failsafe.with(retryPolicy)
|
||||
.onFailure(e -> {
|
||||
if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
|
||||
logger.info("Job successfully interrupted");
|
||||
return;
|
||||
}
|
||||
logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
|
||||
finalTake.getImage().setStatus(Image.Status.ERROR);
|
||||
})
|
||||
.onComplete(e -> {
|
||||
appStateService.doneDownloadJob(finalTake.getImage());
|
||||
logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
|
||||
synchronized (threadCount) {
|
||||
threadCount.decrementAndGet();
|
||||
running.remove(finalTake);
|
||||
futures.remove(finalTake.getImage().getUrl());
|
||||
threadCount.notify();
|
||||
}
|
||||
})
|
||||
.get(finalTake::call);
|
||||
};
|
||||
logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
|
||||
futures.put(finalTake.getImage().getUrl(), executor.submit(task));
|
||||
} else {
|
||||
synchronized (threadCount) {
|
||||
try {
|
||||
threadCount.wait(2_000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int runningCount() {
|
||||
return running.size();
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.q;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class ImageFileData {
|
||||
|
||||
private String pageUrl;
|
||||
private String imageName;
|
||||
private String imageUrl;
|
||||
private HttpUriRequest imageRequest;
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.exception.ValidationException;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.InvalidPathException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.prefs.BackingStoreException;
|
||||
import java.util.prefs.Preferences;
|
||||
|
||||
@Service
|
||||
@Getter
|
||||
@Setter
|
||||
public class AppSettingsService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(AppSettingsService.class);
|
||||
|
||||
private Preferences prefs = Preferences.userNodeForPackage(AppSettingsService.class);
|
||||
|
||||
private final String DOWNLOAD_PATH = "DOWNLOAD_PATH";
|
||||
private final String MAX_THREADS = "MAX_THREADS";
|
||||
private final String AUTO_START = "AUTO_START";
|
||||
private final String V_LOGIN = "VLOGIN";
|
||||
private final String V_USERNAME = "VUSERNAME";
|
||||
private final String V_PASSWORD = "VPASSWORD";
|
||||
private final String V_THANKS = "VTHANKS";
|
||||
private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
|
||||
private final String FORCE_ORDER = "FORCE_ORDER";
|
||||
private final String SUBFOLDER = "SUBFOLDER";
|
||||
private final String THREADSUBFOLDER = "THREADSUBFOLDER";
|
||||
private final String CLEAR = "CLEAR";
|
||||
private final String DARK_THEME = "DARK_THEME";
|
||||
|
||||
private String downloadPath;
|
||||
private int maxThreads;
|
||||
private boolean autoStart;
|
||||
private boolean vLogin;
|
||||
private String vUsername;
|
||||
private String vPassword;
|
||||
private boolean vThanks;
|
||||
private boolean desktopClipboard;
|
||||
private boolean subLocation;
|
||||
private boolean threadSubLocation;
|
||||
private boolean forceOrder;
|
||||
private boolean clearCompleted;
|
||||
private boolean darkTheme;
|
||||
|
||||
public void setVPassword(String vPassword) {
|
||||
if(vPassword.isEmpty()) {
|
||||
this.vPassword = "";
|
||||
} else {
|
||||
this.vPassword = DigestUtils.md5Hex(vPassword);
|
||||
}
|
||||
}
|
||||
|
||||
public void restore() {
|
||||
|
||||
downloadPath = prefs.get(DOWNLOAD_PATH, System.getProperty("user.dir"));
|
||||
maxThreads = prefs.getInt(MAX_THREADS, 4);
|
||||
autoStart = prefs.getBoolean(AUTO_START, true);
|
||||
vLogin = prefs.getBoolean(V_LOGIN, false);
|
||||
vUsername = prefs.get(V_USERNAME, "");
|
||||
vPassword = prefs.get(V_PASSWORD, "");
|
||||
vThanks = prefs.getBoolean(V_THANKS, false);
|
||||
desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
|
||||
forceOrder = prefs.getBoolean(FORCE_ORDER, false);
|
||||
subLocation = prefs.getBoolean(SUBFOLDER, false);
|
||||
threadSubLocation = prefs.getBoolean(THREADSUBFOLDER, false);
|
||||
clearCompleted = prefs.getBoolean(CLEAR, false);
|
||||
darkTheme = prefs.getBoolean(DARK_THEME, false);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void save() {
|
||||
|
||||
prefs.put(DOWNLOAD_PATH, downloadPath);
|
||||
prefs.putInt(MAX_THREADS, maxThreads);
|
||||
prefs.putBoolean(AUTO_START, autoStart);
|
||||
prefs.putBoolean(V_LOGIN, vLogin);
|
||||
prefs.put(V_USERNAME, vUsername);
|
||||
prefs.put(V_PASSWORD, vPassword);
|
||||
prefs.putBoolean(V_THANKS, vThanks);
|
||||
prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
|
||||
prefs.putBoolean(FORCE_ORDER, forceOrder);
|
||||
prefs.putBoolean(SUBFOLDER, subLocation);
|
||||
prefs.putBoolean(THREADSUBFOLDER, threadSubLocation);
|
||||
prefs.putBoolean(CLEAR, clearCompleted);
|
||||
prefs.putBoolean(DARK_THEME, darkTheme);
|
||||
|
||||
try {
|
||||
prefs.sync();
|
||||
} catch (BackingStoreException e) {
|
||||
logger.error("Failed to store user settings", e);
|
||||
}
|
||||
}
|
||||
|
||||
public void check(Settings settings) throws ValidationException {
|
||||
|
||||
Path path;
|
||||
try {
|
||||
path = Paths.get(settings.getDownloadPath());
|
||||
} catch (InvalidPathException e) {
|
||||
throw new ValidationException(String.format("%s is invalid", settings.getDownloadPath()));
|
||||
}
|
||||
if (!Files.exists(path)) {
|
||||
throw new ValidationException(String.format("%s does not exist", settings.getDownloadPath()));
|
||||
} else if (!Files.isDirectory(path)) {
|
||||
throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
|
||||
}
|
||||
|
||||
if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 8) {
|
||||
throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 8));
|
||||
}
|
||||
}
|
||||
|
||||
public Theme getTheme() {
|
||||
return new Theme(this.darkTheme);
|
||||
}
|
||||
|
||||
public void setTheme(Theme theme) {
|
||||
this.darkTheme = theme.darkTheme;
|
||||
save();
|
||||
}
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public static class Theme {
|
||||
|
||||
@JsonProperty("darkTheme")
|
||||
private boolean darkTheme;
|
||||
|
||||
public Theme(boolean darkTheme) {
|
||||
this.darkTheme = darkTheme;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class Settings {
|
||||
|
||||
@JsonProperty("downloadPath")
|
||||
private String downloadPath;
|
||||
@JsonProperty("maxThreads")
|
||||
private int maxThreads;
|
||||
@JsonProperty("autoStart")
|
||||
private boolean autoStart;
|
||||
@JsonProperty("vLogin")
|
||||
private boolean vLogin;
|
||||
@JsonProperty("vUsername")
|
||||
private String vUsername;
|
||||
@JsonProperty("vPassword")
|
||||
private String vPassword;
|
||||
@JsonProperty("vThanks")
|
||||
private boolean vThanks;
|
||||
@JsonProperty("desktopClipboard")
|
||||
private boolean desktopClipboard;
|
||||
@JsonProperty("forceOrder")
|
||||
private boolean forceOrder;
|
||||
@JsonProperty("subLocation")
|
||||
private boolean subLocation;
|
||||
@JsonProperty("threadSubLocation")
|
||||
private boolean threadSubLocation;
|
||||
@JsonProperty("clearCompleted")
|
||||
private boolean clearCompleted;
|
||||
|
||||
public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder, boolean subLocation, boolean threadSubLocation, boolean clearCompleted) {
|
||||
this.downloadPath = downloadPath;
|
||||
this.maxThreads = maxThreads;
|
||||
this.autoStart = autoStart;
|
||||
this.vLogin = vLogin;
|
||||
this.vUsername = vUsername;
|
||||
this.vPassword = vPassword;
|
||||
this.vThanks = vThanks;
|
||||
this.desktopClipboard = desktopClipboard;
|
||||
this.forceOrder = forceOrder;
|
||||
this.subLocation = subLocation;
|
||||
this.threadSubLocation = threadSubLocation;
|
||||
this.clearCompleted = clearCompleted;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import io.reactivex.disposables.Disposable;
|
||||
import io.reactivex.processors.PublishProcessor;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.entities.Image;
|
||||
import tn.mnlr.vripper.entities.Post;
|
||||
import tn.mnlr.vripper.q.DownloadJob;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Getter
|
||||
public class AppStateService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(AppStateService.class);
|
||||
|
||||
private Map<String, Image> currentImages = new ConcurrentHashMap<>();
|
||||
|
||||
private Map<String, Post> currentPosts = new ConcurrentHashMap<>();
|
||||
|
||||
private Map<String, AtomicInteger> runningPosts = new ConcurrentHashMap<>();
|
||||
|
||||
private PublishProcessor<Image> liveImageUpdates = PublishProcessor.create();
|
||||
|
||||
private PublishProcessor<Post> livePostsState = PublishProcessor.create();
|
||||
|
||||
@Autowired
|
||||
private PersistenceService persistenceService;
|
||||
|
||||
@Autowired
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
private Disposable subscription;
|
||||
|
||||
public void onImageUpdate(Image imageState) {
|
||||
|
||||
persistenceService.getProcessor().onNext(currentPosts);
|
||||
|
||||
liveImageUpdates.onNext(imageState);
|
||||
if (imageState.isCompleted()) {
|
||||
|
||||
Post postState = currentPosts.get(imageState.getPostId());
|
||||
postState.increase();
|
||||
}
|
||||
}
|
||||
|
||||
public Post getPost(String postId) {
|
||||
return currentPosts.get(postId);
|
||||
}
|
||||
|
||||
public synchronized void newDownloadJob(DownloadJob downloadJob) {
|
||||
String postId = downloadJob.getImage().getPostId();
|
||||
checkKeyRunningPosts(postId);
|
||||
runningPosts.get(postId).incrementAndGet();
|
||||
}
|
||||
|
||||
public synchronized void postDownloadingUpdate(String postId) {
|
||||
Post post = currentPosts.get(postId);
|
||||
if (!post.getStatus().equals(Post.Status.DOWNLOADING)) {
|
||||
post.setStatus(Post.Status.DOWNLOADING);
|
||||
livePostsState.onNext(post);
|
||||
}
|
||||
}
|
||||
|
||||
public void doneDownloadJob(Image image) {
|
||||
String postId = image.getPostId();
|
||||
int i = runningPosts.get(postId).decrementAndGet();
|
||||
Post post = currentPosts.get(postId);
|
||||
if(image.getStatus().equals(Image.Status.ERROR)) {
|
||||
post.setStatus(Post.Status.PARTIAL);
|
||||
}
|
||||
if (i == 0) {
|
||||
if (post.getImages().stream().map(Image::getStatus).filter(e -> e.equals(Image.Status.ERROR)).count() > 0) {
|
||||
post.setStatus(Post.Status.ERROR);
|
||||
} else {
|
||||
if (!Post.Status.STOPPED.equals(post.getStatus())) {
|
||||
post.setStatus(Post.Status.COMPLETE);
|
||||
if (appSettingsService.isClearCompleted()) {
|
||||
remove(image.getPostId());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void checkKeyRunningPosts(String key) {
|
||||
if (!runningPosts.containsKey(key)) {
|
||||
runningPosts.put(key, new AtomicInteger(0));
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void remove(String postId) {
|
||||
currentPosts.get(postId).setRemoved(true);
|
||||
runningPosts.remove(postId);
|
||||
currentPosts.remove(postId);
|
||||
Iterator<Map.Entry<String, Image>> iterator = currentImages.entrySet().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Map.Entry<String, Image> entry = iterator.next();
|
||||
if(entry.getValue().getPostId().equals(postId)) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
persistenceService.getProcessor().onNext(currentPosts);
|
||||
}
|
||||
|
||||
public synchronized List<String> clearAll() {
|
||||
List<String> toRemove = this.currentPosts
|
||||
.values()
|
||||
.stream()
|
||||
.filter(e -> e.getStatus().equals(Post.Status.COMPLETE) && e.getDone().get() >= e.getTotal())
|
||||
.map(Post::getPostId)
|
||||
.collect(Collectors.toList());
|
||||
toRemove.forEach(this::remove);
|
||||
return toRemove;
|
||||
}
|
||||
|
||||
public synchronized List<String> removeAll() {
|
||||
List<String> toRemove = this.currentPosts
|
||||
.values()
|
||||
.stream()
|
||||
.map(Post::getPostId)
|
||||
.collect(Collectors.toList());
|
||||
toRemove.forEach(this::remove);
|
||||
return toRemove;
|
||||
}
|
||||
|
||||
|
||||
@Getter
|
||||
public static class CachedThread {
|
||||
|
||||
private Map<String, Post> posts = new ConcurrentHashMap<>();
|
||||
private AtomicInteger parsed = new AtomicInteger(0);
|
||||
|
||||
@Setter
|
||||
private int total = 0;
|
||||
}
|
||||
}
|
||||