Compare commits

...
70 Commits
Author SHA1 Message Date
dev-claw a8e5dbf5f7 change package id 2025-09-29 17:50:02 +01:00
dev-claw 70cee5cb17 change artifact id 2025-09-29 17:46:48 +01:00
dev-claw 9012cc065e backport fixes for Imx and ImageBam 2025-09-29 17:37:50 +01:00
death-claw c018033d2f Code enhancements 2021-05-31 20:22:16 +01:00
death-claw b0f5a15943 Add support for dpic.me 2021-05-31 19:06:41 +01:00
death-claw 8db4d406ad v3.5.4 2021-05-29 10:32:26 +01:00
death-claw 47f1705abb Fix download queue bug 2021-05-29 10:30:18 +01:00
death-claw 816fb84549 v3.5.3 2021-05-24 15:17:30 +01:00
death-claw 5213ed7129 Fix icon 2021-05-24 15:15:49 +01:00
death-claw 2483c5fa00 v3.5.2 2021-05-24 14:59:32 +01:00
death-claw dfc5810fd3 Add missing icon 2021-05-24 14:57:47 +01:00
death-claw fad418134e Update donation infos 2021-05-24 14:56:00 +01:00
death-claw 9dd91bfc07 v3.5.1 2021-05-22 15:51:05 +01:00
death-claw 5bd42e2ce5 Merge remote-tracking branch 'origin/master' 2021-05-22 15:44:22 +01:00
death-claw e18832b154 v3.5.0 2021-05-21 20:11:16 +01:00
death-claw 41feb53a83 Bug fix 2021-05-20 23:46:59 +01:00
death-claw 6f166b42c4 Enhance metadata interruption
Add Added on and Order columns
2021-05-20 21:25:52 +01:00
death-claw e8ee579a37 Save window size and position
Resize confirmation box
2021-05-18 21:10:42 +01:00
death-claw 41ce99995b Download service enhancements 2021-05-10 21:19:40 +01:00
death-claw 5caeef0e70 v3.4.2 2021-05-10 11:18:43 +01:00
death-claw 96ef0cb234 Fix race condition 2021-05-10 11:17:02 +01:00
death-claw 641b3e3e7b v3.4.1 2021-05-10 00:21:23 +01:00
death-claw 08de4dc5b0 Fix icons 2021-05-10 00:19:42 +01:00
death-claw 57680a6053 v3.4.0 2021-05-09 22:45:54 +01:00
death-claw 53fd14935b Enhance download queue logic 2021-05-09 22:39:22 +01:00
death-claw 90af03d68f Fix imagebam
Code enhancements
2021-05-09 14:12:48 +01:00
death-claw 265614585b Parameterize settings 2021-05-09 10:35:41 +01:00
death-claw 4503d540ed Implement an event bus 2021-05-09 00:17:32 +01:00
death-claw b08b730dff replace guava with caffeine 2021-05-08 13:03:20 +01:00
death-claw 06d636329e v3.3.8 2021-05-05 18:31:13 +01:00
death-claw 85313600fc Fix fonts path 2021-05-05 18:29:50 +01:00
death-claw 7d451397f9 v3.3.7 2021-05-05 18:16:00 +01:00
death-claw 1c94b93597 Fix imagebam host and code format 2021-05-05 18:14:25 +01:00
death-claw 8f09596453 Fix electron build 2021-03-20 11:35:46 +01:00
death-claw 6ce05e78b3 v3.3.6 2021-03-20 11:28:20 +01:00
death-claw 67a375150c Optimize build size 2021-03-20 11:27:06 +01:00
death-claw fba5f670a0 v3.3.5 2021-03-18 09:31:22 +01:00
death-claw 7d083630cc Bug fix 2021-03-18 09:29:45 +01:00
death-claw 335cbf70b4 v3.3.4 2021-03-18 09:12:48 +01:00
death-claw e25d306f1d Bug fixes 2021-03-18 09:11:01 +01:00
death-claw f395338a9d Update build plugins 2021-03-13 17:08:13 +01:00
death-claw 58a40682cc v3.3.3 2021-03-13 16:21:36 +01:00
death-claw ccc4032bb0 Remove unnecessary event logs 2021-03-13 16:15:56 +01:00
death-claw 1c5b8a0397 v3.3.2 2021-03-11 19:05:09 +01:00
death-claw cc2ce9482d Fix a bug with event logs
Change event names and add some icons
2021-03-11 19:02:01 +01:00
death-claw 41f1775360 v3.3.1 2021-02-18 16:04:34 +01:00
death-claw 733f1a08ac Add about menu
Fix README.md
2021-02-18 16:01:06 +01:00
death-claw 0cb0ee0100 v3.3.0 2021-02-16 19:15:58 +01:00
death-claw cb88e39c83 Add support for proxies
Add a Event Logs
Fix bugs with the database
2021-02-16 19:13:02 +01:00
death-claw 6375c0aed6 v3.2.3 2021-01-13 20:15:13 +01:00
death-claw 40ca4cf86b Finished posts are not cleared from the UI 2021-01-13 20:11:47 +01:00
death-clawandGitHub b3a798388b Update README.md 2021-01-13 12:54:51 +01:00
death-claw 6e69b2bce8 v3.2.2 2021-01-13 09:51:56 +01:00
death-claw bb6dcd77b9 Set total download to max value 2021-01-13 09:49:38 +01:00
death-claw 42e1aa73e2 v3.2.1 2021-01-12 20:02:17 +01:00
death-claw 26fde22af2 Code reformat 2021-01-12 19:37:12 +01:00
death-claw 798f39d54f Add settings for connection timeout
Add settings for maximum attempts
Enhance posts rename logic
Dependencies update
Switch reactive from rx-java to reactor
2021-01-12 19:32:43 +01:00
death-claw 3a21dfac38 Remove unused font 2021-01-06 14:55:49 +01:00
death-claw e8355b88b9 v3.2 2020-11-16 19:40:36 +01:00
death-claw 379660236d Commit missing files 2020-11-16 19:31:59 +01:00
death-claw c86ee91c27 v3.1.0 2020-11-16 16:44:26 +01:00
death-claw ff9d253242 UI revamp
Fix bugs
2020-11-16 16:42:30 +01:00
death-claw 53b45e00ba v3.0.11 2020-10-31 15:09:19 +01:00
death-claw 247b12a525 Add support for vipr.im 2020-10-31 15:06:26 +01:00
death-claw 7a8d332e6e v3.0.10 2020-10-04 16:31:57 +01:00
death-claw 749a6f18d2 Fix bug on settings update 2020-10-04 16:27:26 +01:00
death-claw 4107048f12 v3.0.9 2020-09-27 10:14:08 +01:00
death-claw f5d9ab85a2 Revert to appending post id to download folders
Fix a bug that lead to adding duplicate posts
Remove old unused settings
2020-09-27 10:12:29 +01:00
death-claw b82ee7b494 v3.0.8 2020-08-22 19:01:08 +01:00
death-claw 8a31dfc012 Bug fix which breaks concurrent downloads 2020-08-22 18:59:44 +01:00
340 changed files with 21909 additions and 24463 deletions
+347 -3
View File
@@ -1,177 +1,417 @@
# Changelog
## [3.0.7] - 2020-08-22
## [3.5.4] - 2021-05-29
### Changed
- Fix download queue bug
## [3.5.3] - 2021-05-24
### Changed
- Update donation infos
## [3.5.1] - 2021-05-22
### Changed
- Bug fix
## [3.5.0] - 2021-05-21
### Changed
- Download queue bug fix
- Save window size and position
- Resize confirmation box
- Enhance metadata interruption
- Add Added on and Order columns
## [3.4.2] - 2021-05-10
### Changed
- Fix potential issue with imagebam host
- Enhance download queue logic
- Code source enhancements
- Fix app icons
- Fix race condition bug
## [3.3.8] - 2021-05-05
### Changed
- Fix imagebam host
- Fix font path
## [3.3.6] - 2021-03-20
### Changed
- Optimize build size
## [3.3.5] - 2021-03-18
### Changed
- Bug fixe
## [3.3.4] - 2021-03-18
### Changed
- Bug fixes
## [3.3.3] - 2021-03-13
### Changed
- Remove unnecessary event logs
## [3.3.2] - 2021-03-11
### Changed
- Fix a bug with event logs
- Change event names and add some icons
## [3.3.1] - 2021-02-18
### Added
- Add about menu
### Changed
- Fix README.md
## [3.3.0] - 2021-02-16
### Changed
- Add support for proxies
- Add a Event Logs
- Fix bugs with the database
## [3.2.3] - 2021-01-13
### Changed
- Finished posts are not cleared from the UI
## [3.2.2] - 2021-01-13
### Changed
- Set total download to max value
## [3.2.1] - 2021-01-12
### Added
- Add settings for connection timeout
- Add settings for maximum attempts
### Changed
- Enhance posts rename logic
- Dependencies update
- Switch reactive from rx-java to reactor
## [3.2.0] - 2020-11-16
### Changed
- Fix build issue
## [3.1.0] - 2020-11-16
### Changed
- UI revamp
- Fix bugs
## [3.0.11] - 2020-10-31
### Added
- Support for vipr.im
## [3.0.10] - 2020-10-04
### Changed
- Fix bug on settings update
## [3.0.9] - 2020-09-27
### Changed
- Revert to appending post id to download folders
- Fix a bug that lead to adding duplicate posts
- Remove old unused settings
## [3.0.8] - 2020-08-22
### Changed
- Bug fix which breaks concurrent downloads
## [3.0.7] - 2020-08-22
### Changed
- Fix MacOs build
## [3.0.6] - 2020-08-18
### Changed
- Fix TurboImageHost
## [3.0.5] - 2020-08-17
### Changed
- Update packages
- Host class refactoring
## [3.0.4] - 2020-08-16
### Changed
- Improve rename UI
## [3.0.3] - 2020-08-16
### Changed
- Fix bugs with electron app
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
- Introducing a proper database for persistence
## [2.12.1] - 2020-05-22
### Changed
- Fix alternative name bug
## [2.12.0] - 2020-05-21
### Changed
- Use new user hash from the api
## [2.11.7] - 2020-05-08
### Changed
- Fix bug with paths creation
- Move Open Link button
- Rename rename all title
## [2.11.6] - 2020-05-04
### Changed
- PNG filenames are now correctly named
- Fix issue with special characters in post titles
## [2.11.5] - 2020-05-03
### Changed
- Reduce rows buffer size
## [2.11.4] - 2020-05-03
### Changed
- Code cleanup
- Dependencies update
## [2.11.3] - 2020-04-30
### Changed
- Source code restructure
- UI fixes and enhancements
### Added
- Add menu entry to rename a gallery to first alternative title
- Add menu on toolbar for massive rename to first alternative title
## [2.11.2] - 2020-04-26
### Changed
- Bug fixes
## [2.11.1] - 2020-04-25
### Changed
- Code source maintenance
### Added
- Added option to rename gallery
## [2.11.0] - 2020-03-29
### Changed
- Upgrade Electron to v8
- Upgrade Angular to v9
- App settings are now portable in a json file
- General code source maintenance
### Added
- Added Support for pixroute.com
- Added grab post title from thread option
## [2.10.8] - 2020-01-25
### Changed
- Fix app url
## [2.10.7] - 2020-01-25
### Changed
- Drop SockJs and switch to WebSocket
- Autoreconnect front to back
## [2.10.6] - 2020-01-19
### Changed
- Limit cache size
## [2.10.5] - 2020-01-18
### Changed
- Bug fixes
## [2.10.4] - 2020-01-18
### Changed
- Zero value for total max download will disable the limit
- Add system notification
- Bug fixes
## [2.10.3] - 2020-01-16
### Changed
- Some bug fixes
- Enhance download queue logic
- Clear cache button
- Fix the partial status
- Sanitize URLs with spaces
### Added
- Support for Postimg host
- Support for Imagevenue host
## [2.10.2] - 2019-12-29
### Changed
- Upgrade electron to v7
- Bug fix
## [2.10.1] - 2019-12-27
### Changed
- Bug fix
## [2.10.0] - 2019-12-27
### Changed
- Fix spring dependency bug
### Added
- Add support for imgspice
## [2.9.0] - 2019-12-27
### Changed
- Fix build issue
## [2.9.0] - 2019-12-27
### Changed
- Decrease timeout for fast fail
### Added
- Allow many hosts to run at the same time
- Add title to gallery
- Add global concurrent downloads
- Add counter badge in link collector
## [2.8.1] - 2019-12-24
### Changed
- Fix issue with path creation
## [2.8.0] - 2019-12-24
### Added
- Photo gallery feature
## [2.7.1] - 2019-12-21
### Added
- Add version to title bar
## [2.7.0] - 2019-12-21
### Changed
- Add missing icon
### Added
- Add support for pixxxels
## [2.6.0] - 2019-12-21
### Changed
- Disable download button when nothing selected in Link collector grab menu
- Better quality app icon
- Implement context menu on download tab
@@ -180,143 +420,207 @@
- Some UI fixes
## [2.5.1] - 2019-12-09
### Changed
- Disable download button when nothing selected in Link collector grab menu
### Added
- Add hosts column in Link collector grab menu
## [2.5.0] - 2019-12-08
### Changed
- Change loading screen
- UI Fixes
- Fix bug with previews
### Added
- Add link collector feature
- Add support for multiple links at a time
## [2.4.1] - 2019-11-26
### Changed
- Increase attempts for establishing ws connection between front and back end
- Change default download folder to Home folder (installer), App folder (portable)
## [2.4.0] - 2019-11-23
### Changed
- Pimpandhost Host support
## [2.3.2] - 2019-11-23
### Changed
- Fix a bug with the scan component
## [2.3.1] - 2019-11-22
### Changed
- Fix connexion reset issue
- Fix Post Status when autostart option is selected
- Add stop all button
## [2.3.0] - 2019-11-21
### Changed
- UI enhancements and overhaul
- Stability fixes with HTTP connections
- MacOS official support
- Use local sockjs and material icons font
## [2.2.5] - 2019-11-09
### Changed
- Exclude thumbs download for imagezilla and pixhost
- Change retry on fail logic (may help avoid having download errors)
## [2.2.4] - 2019-10-05
### Added
- AppImage target for linux
## [2.2.3] - 2019-10-01
### Changed
- Strip libjvm.so for debian
## [2.2.2] - 2019-10-01
### Changed
- Update product name
## [2.2.1] - 2019-10-01
### Changed
- Update dependency list for deb package
## [2.2.0] - 2019-10-01
### Added
- Embedd JRE within the app
### Changed
- Fix 'Start' on pending items
- Upgrade to Java 11
## [2.1.0] - 2019-09-30
### Added
- Add a create a subfolder per thread option
## [2.0.4] - 2019-09-27
### Changed
- Fix some issues with ImxHost
## [2.0.3] - 2019-09-23
### Changed
- Update deb dependencies
## [2.0.2] - 2019-09-23
### Changed
- Add portable mode
## [2.0.1] - 2019-09-22
### Changed
- Change app name
## [2.0.0] - 2019-09-22
### Added
- Add sub folder option
- Add clear finished option
### Changed
- UI enhancements
- Bug fixes
- Change data and log paths
## [1.6.5] - 2019-09-18
### Changed
- electron update
## [1.6.4] - 2019-09-18
### Changed
- Change menu item name
## [1.6.3] - 2019-09-18
### Changed
- Better multithread management
- Fix partial state UI
## [1.6.2] - 2019-09-17
### Changed
- Rework UI colors
- Fix scan input bug
## [1.6.1] - 2019-09-16
### Changed
- Rework UI
- Fix clipboard monitoring
## [1.6.0] - 2019-09-15
### Changed
- Rework the UI
- Add a dark theme
- Fix the force ordering bug
- Other bug fixes
## [1.5.2] - 2019-09-01
### Added
- imx.to host fix
## [1.5.1] - 2019-08-24
### Added
- Bug fixes
## [1.5.0] - 2019-08-24
### Added
- Display logged in user
- Add post details when clicking on main list items (Status, Post URL, Host)
- Add open download location on main list items menu
@@ -324,86 +628,126 @@
- Add link to image from details list
## [1.4.10] - 2019-08-15
### Changed
- Fix a rename issue on windows
## [1.4.9] - 2019-08-13
### Changed
- Update Built-By to be generic
## [1.4.8] - 2019-08-13
### Changed
- Add option to force image ordering
## [1.4.7] - 2019-08-13
### Changed
- Fixed a bug in image filename extensions
- Fixed a bug in image filename extensions
## [1.4.6] - 2019-08-06
### Changed
- Fixed a bug when creating image file names
- Remove unnecessary logs
- Fix data.json init
- Fix data.json init
## [1.4.5] - 2019-08-03
### Changed
- Add min width and height for electron app
### Added
- Add download speed to status bar
## [1.4.4] - 2019-08-01
### Changed
- Fix names for ImgboxHost
- Enhance UI
- Minor fixes
### Added
- Add start and stop button
## [1.4.3] - 2019-07-14
### Changed
- Update some messages
- Fix scrolling issue
## [1.4.2] - 2019-07-14
### Changed
- Fix layout issue in web mode
## [1.4.1] - 2019-07-14
### Changed
- Change title's font
### Added
- Add borders
## [1.4.0] - 2019-07-14
### Changed
- Better error handling on startup
- Better support for linux AppImage
### Added
- Clear all completed
- Add clipboard support
- Add option to remove all posts
- Add confirmation on remove
## [1.3.0-rc0] - 2019-07-04
### Changed
- Change default setting for autostart false -> true
- Change default setting for max concurrent downloads 1 -> 4
### Added
- Desktop app built with electron
- Add the option to remove a post
## [1.2.0] - 2019-06-23
### Added
- Add a shutdown button to gracefully shutdown the app
- Open and redirect the browser to the app address on startup
- Add the option to run the app in headless mode (-Djava.awt.headless=true)
## [1.1.1] - 2019-06-22
### Changed
- Fix a stability issue with progress updates when the app is runninh for a long time
## [1.0.0] - 2019-06-22
### Added
- Add real time communication for progress updates via websocket (SockJS)
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
+19 -8
View File
@@ -1,24 +1,35 @@
# Viper Ripper!
# VRipper!
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website.
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website. The purpose of
this project is to build a robust and clean application to conveniently download photo galleries using modern web
technologies, Java + Spring boot for the back end and angular + electron for the front end.
## How to build
You need a recent version of maven 3.6.1+.
The application support 2 build modes:
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there
for you.
To build the server app:
mvn clean install -DskipTests
To build the desktop app:
mvn clean install -Pelectron -DskipTests
Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
Maven will automatically handle front end compilation. However, for development, you will need to install a recent
version of nodejs on your system.
[![Donate with PayPal](https://raw.githubusercontent.com/stefan-niedermann/paypal-donate-button/master/paypal-donate-button.png)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=SC7M5LNWZ528Q)
If you feel generous and want to support me, you can donate some coins, and I would be very grateful 🙏
I accept [PayPal](https://www.paypal.com/myaccount/transfer/homepage/buy/preview) donations, following is my email
address
dev.vripper@gmail.com
+6 -6
View File
@@ -2,19 +2,19 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.7</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.0.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
<version>3.5.6</version>
<relativePath/>
</parent>
<modules>
<module>vripper-server</module>
<module>vripper-ui</module>
<module>vripper-electron</module>
</modules>
</project>
</project>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

+16 -5
View File
@@ -7,6 +7,7 @@ const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const windowStateKeeper = require('electron-window-state');
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
@@ -29,18 +30,26 @@ process.on("uncaughtException", err => {
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId("tn.mnlr.vripper");
app.setAppUserModelId('me.vripper.vripper-app');
}
let icon;
if (process.platform === "win32") {
if (process.platform === 'win32') {
icon = __dirname + '/icon.ico';
} else {
} else if (process.platform === 'linux') {
icon = __dirname + '/icon.png';
}
const mainWindowState = windowStateKeeper({
defaultWidth: 1024,
defaultHeight: 800
});
win = new BrowserWindow({
width: 1024,
height: 800,
x: mainWindowState.x,
y: mainWindowState.y,
width: mainWindowState.width,
height: mainWindowState.height,
minWidth: 800,
minHeight: 600,
webPreferences: {
@@ -50,6 +59,7 @@ createWindow = () => {
icon: icon
});
mainWindowState.manage(win);
win.removeMenu();
win.setMenu(null);
win.setMenuBarVisibility(false);
@@ -130,6 +140,7 @@ if (!gotTheLock) {
"-Xms256m",
"-Dvripper.server.port=" + port,
"-Dbase.dir=" + baseDir,
"-Djava.net.preferIPv4Stack=true",
"-jar",
jarPath
], {
-2519
View File
File diff suppressed because it is too large Load Diff
+12 -8
View File
@@ -1,13 +1,13 @@
{
"name": "vripper-electron",
"version": "3.0.7",
"version": "3.6.0",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
"homepage": "https://github.com/death-claw/vripper-project",
"author": "dev-claw",
"homepage": "https://github.com/dev-claw/vripper-project",
"license": "ISC",
"build": {
"appId": "tn.mnlr.vripper",
"appId": "me.vripper.vripper-app",
"productName": "VRipper",
"files": [
"**/*",
@@ -22,6 +22,10 @@
"!**/{__pycache__,thumbs.db,.flowconfig,.idea,.vs,.nyc_output}",
"!**/{appveyor.yml,.travis.yml,circle.yml}",
"!**/{npm-debug.log,yarn.lock,.yarn-integrity,.yarn-metadata.json}",
"!**/build-dir",
"!**/java-runtime",
"!**/pre-build.js",
"!**/pom.xml",
{
"from": "./build/",
"to": "."
@@ -47,7 +51,6 @@
"synopsis": "vipergirls.to ripper",
"category": "Utility",
"packageCategory": "Utility",
"icon": "icons",
"target": [
"AppImage"
]
@@ -65,14 +68,15 @@
"dist": "node pre-build.js && electron-builder"
},
"devDependencies": {
"electron": "9.1.0",
"electron-builder": "22.7.0"
"electron": "11.1.1",
"electron-builder": "22.9.1"
},
"dependencies": {
"axios": "0.19.2",
"axios": "0.21.1",
"cheerio": "1.0.0-rc.3",
"copy-dir": "1.3.0",
"electron-context-menu": "2.1.0",
"electron-window-state": "^5.0.3",
"get-port": "5.1.1",
"rimraf": "3.0.2",
"v8-compile-cache": "2.1.1"
+72 -71
View File
@@ -1,77 +1,78 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.7</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<classifier>${buildClassifier}</classifier>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<packaging>pom</packaging>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.3</version>
<configuration>
<nodeVersion>v12.2.0</nodeVersion>
<npmVersion>6.9.0</npmVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<arguments>run dist</arguments>
<classifier>${buildClassifier}</classifier>
</configuration>
<phase>generate-resources</phase>
</execution>
</executions>
</plugin>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>me.vripper</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.11.2</version>
<configuration>
<nodeVersion>v14.16.0</nodeVersion>
<yarnVersion>v1.22.10</yarnVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and yarn</id>
<goals>
<goal>install-node-and-yarn</goal>
</goals>
</execution>
<execution>
<id>yarn install</id>
<goals>
<goal>yarn</goal>
</goals>
</execution>
<execution>
<id>yarn run build</id>
<goals>
<goal>yarn</goal>
</goals>
<configuration>
<arguments>run dist</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
+4 -4
View File
@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
const fs = require('fs');
const rimraf = require("rimraf");
const copydir = require("copy-dir");
const { execSync } = require("child_process");
const {execSync} = require("child_process");
rimraf.sync('./build/vripper-ui');
copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -18,8 +18,8 @@ fs.writeFileSync('./build/vripper-ui/index.html', $.html());
console.log('Building runtime environment');
rimraf.sync('java-runtime');
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
if(process.platform === 'linux') {
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.security.jgss,java.sql,jdk.unsupported,jdk.crypto.ec --output java-runtime');
if (process.platform === 'linux') {
console.log('Stripping libjvm.so');
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
}
}
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
File diff suppressed because it is too large Load Diff
+34 -67
View File
@@ -1,11 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.0.7</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
@@ -16,14 +16,6 @@
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jersey</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
@@ -43,7 +35,6 @@
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.5.1</version>
<scope>runtime</scope>
</dependency>
<dependency>
@@ -64,17 +55,16 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
<version>4.5.14</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.22</version>
<version>2.24</version>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
<artifactId>rxjava</artifactId>
<version>2.2.6</version>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>net.jodah</groupId>
@@ -82,25 +72,45 @@
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>28.1-jre</version>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>tn.mnlr</groupId>
<groupId>me.vripper</groupId>
<artifactId>vripper-ui</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<type>pom</type>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.14.0</version>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.4.2</version>
<configuration>
<classifier>${buildClassifier}</classifier>
<archive>
<manifestEntries>
<Build-Time>${maven.build.timestamp}</Build-Time>
<Built-By>${user.name}</Built-By>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
@@ -118,53 +128,10 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
</execution>
<execution>
<id>validate-the-git-infos</id>
<goals>
<goal>validateRevision</goal>
</goals>
</execution>
</executions>
<configuration>
<dotGitDirectory>${project.parent.basedir}/.git</dotGitDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<archive>
<manifestEntries>
<Build-Time>${maven.build.timestamp}</Build-Time>
<Built-By>${user.name}</Built-By>
</manifestEntries>
<manifestSections>
<manifestSection>
<name>Versions</name>
<manifestEntries>
<Implementation-Version>${git.build.version}</Implementation-Version>
<Implementation-SCM-Revision>${git.commit.id}</Implementation-SCM-Revision>
<Implementation-SCM-Branch>${git.branch}</Implementation-SCM-Branch>
<Implementation-SCM-Time>${git.build.time}</Implementation-SCM-Time>
<Implementation-SCM-Dirty>${git.dirty}</Implementation-SCM-Dirty>
<Implementation-SCM-Tags>${git.tags}</Implementation-SCM-Tags>
</manifestEntries>
</manifestSection>
</manifestSections>
</archive>
</configuration>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
</plugins>
</build>
@@ -0,0 +1,31 @@
package me.vripper;
import lombok.Getter;
import me.vripper.jpa.repositories.IPostRepository;
import me.vripper.services.DataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
@Component
public class EventListenerBean {
@Getter private static boolean init = false;
private final IPostRepository postRepository;
private final DataService dataService;
@Autowired
public EventListenerBean(IPostRepository postRepository, DataService dataService) {
this.postRepository = postRepository;
this.dataService = dataService;
}
@EventListener
public void onApplicationEvent(ContextRefreshedEvent event) {
postRepository.setDownloadingToStopped();
dataService.sortPostsByRank();
init = true;
}
}
@@ -0,0 +1,38 @@
package me.vripper;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class SpringContext implements ApplicationContextAware {
private static ConfigurableApplicationContext context;
public static <T> T getBean(Class<T> beanClass) {
return context.getBean(beanClass);
}
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
return context.getBeansOfType(beanClass);
}
public static void close() {
log.info("Application terminating...");
if (context != null) {
context.close();
}
}
@Override
public void setApplicationContext(ApplicationContext context) throws BeansException {
// store ApplicationContext reference to access required beans later on
SpringContext.context = ((ConfigurableApplicationContext) context);
}
}
@@ -0,0 +1,23 @@
package me.vripper;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Utils {
public static String throwableToString(Throwable th) {
try (StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter)) {
th.printStackTrace(printWriter);
printWriter.flush();
stringWriter.flush();
return stringWriter.toString();
} catch (IOException e) {
log.error("Failed to convert throwable to string", e);
}
return "Cannot display error, please check the log file for details";
}
}
@@ -0,0 +1,19 @@
package me.vripper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@Slf4j
public class VripperApplication {
public static void main(String[] args) {
try {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
SpringApplication.run(VripperApplication.class, args);
} catch (Exception e) {
log.error("Failed to run the application", e);
}
}
}
@@ -0,0 +1,294 @@
package me.vripper.download;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.services.*;
import me.vripper.services.domain.Settings;
import net.jodah.failsafe.function.CheckedRunnable;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
@Slf4j
public class DownloadJob implements CheckedRunnable {
private static final Object LOCK = new Object();
private static final int READ_BUFFER_SIZE = 8192;
private final DataService dataService;
private final PathService pathService;
private final ConnectionService cm;
private final VGAuthService authService;
private final DownloadSpeedService downloadSpeedService;
private final Settings settings;
private final HttpClientContext context;
@Getter private final Image image;
@Getter private final Post post;
private volatile boolean stopped = false;
@Getter private volatile boolean finished = false;
DownloadJob(Post post, Image image, Settings settings) {
this.image = image;
this.post = post;
this.settings = settings;
dataService = SpringContext.getBean(DataService.class);
pathService = SpringContext.getBean(PathService.class);
cm = SpringContext.getBean(ConnectionService.class);
authService = SpringContext.getBean(VGAuthService.class);
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
context.setAttribute(
ContextAttributes.OPEN_CONNECTION.toString(),
Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
}
public void download(final Post post, final Image image) throws DownloadException {
try {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING)
&& !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
// The post may be updated and the download directory might be set by another thread
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
if (updatedPost.getDownloadDirectory() == null) {
pathService.createDefaultPostFolder(updatedPost, settings);
}
if (settings.getLeaveThanksOnStart() != null && settings.getLeaveThanksOnStart()) {
authService.leaveThanks(updatedPost);
}
}
if (stopped) {
return;
}
/*
* HOST SPECIFIC
*/
log.debug(
String.format(
"Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image, context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
log.debug(
String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
/*
* END HOST SPECIFIC
*/
if (stopped) {
return;
}
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
log.debug(
String.format(
"Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
HttpClient client = cm.getClient().build();
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
httpGet.addHeader("Referer", image.getUrl());
try (CloseableHttpResponse response =
(CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(
String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (stopped) {
return;
}
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
try (InputStream downloadStream = response.getEntity().getContent();
FileOutputStream fos = new FileOutputStream(outputFile)) {
if (stopped) {
return;
}
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
if (stopped) {
return;
}
}
checkImageTypeAndRename(
dataService.findById(post.getId()).orElseThrow(),
outputFile,
nameAndUrl.getName(),
image.getIndex());
}
} catch (Exception e) {
if (stopped) {
return;
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
image.setStatus(Status.COMPLETE);
} else if (stopped) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
finished = true;
}
}
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index)
throws HostException {
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
imageName += ".jpg";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_jpg")) {
toReplace = "_jpg";
} else if (imageNameLC.endsWith("_jpeg")) {
toReplace = "_jpeg";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
}
}
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_png")) {
imageName += ".png";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_png")) {
toReplace = "_png";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
}
}
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
pathService.getDirectoryAccess().lock();
File downloadDestinationFolder = pathService.calcDownloadDirectory(post, settings);
File outImage =
new File(
downloadDestinationFolder,
(settings.getForceOrder() ? String.format("%03d_", index) : "") + imageName);
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
} finally {
pathService.getDirectoryAccess().unlock();
try {
Files.delete(outputFile.toPath());
} catch (IOException e) {
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
}
}
}
@Override
public void run() throws Exception {
if (stopped) {
finished = true;
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
download(post, image);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return Objects.equals(image, that.image) && Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image, post);
}
public void stop() {
this.stopped = true;
List<AbstractExecutionAwareRequest> requests =
(List<AbstractExecutionAwareRequest>)
this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
}
public enum ContextAttributes {
OPEN_CONNECTION("OPEN_CONNECTION");
private final String value;
ContextAttributes(String value) {
this.value = value;
}
@Override
public String toString() {
return value;
}
}
}
@@ -0,0 +1,69 @@
package me.vripper.download;
import java.time.LocalDateTime;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.ConnectionService;
import me.vripper.services.DataService;
import net.jodah.failsafe.Failsafe;
@Slf4j
public class DownloadJobWrapper implements Runnable {
private final DownloadService downloadService;
private final DataService dataService;
private final ConnectionService connectionService;
private final ILogEventRepository eventRepository;
private final DownloadJob downloadJob;
public DownloadJobWrapper(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
connectionService = SpringContext.getBean(ConnectionService.class);
eventRepository = SpringContext.getBean(ILogEventRepository.class);
this.downloadJob = downloadJob;
}
@Override
public void run() {
Failsafe.with(connectionService.getRetryPolicy())
.onFailure(
e -> {
try {
LogEvent logEvent =
new LogEvent(
LogEvent.Type.DOWNLOAD,
LogEvent.Status.ERROR,
LocalDateTime.now(),
String.format(
"Failed to download %s\n %s",
downloadJob.getImage().getUrl(),
Utils.throwableToString(e.getFailure())));
eventRepository.save(logEvent);
} catch (Exception exp) {
log.error("Failed to save event", exp);
}
log.error(
String.format(
"Failed to download %s after %d tries",
downloadJob.getImage().getUrl(), e.getAttemptCount()),
e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(
downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
})
.onComplete(
e -> {
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
downloadService.afterJobFinish(downloadJob);
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
})
.run(downloadJob);
}
}
@@ -0,0 +1,273 @@
package me.vripper.download;
import jakarta.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.services.DataService;
import me.vripper.services.MetadataService;
import me.vripper.services.SettingsService;
import me.vripper.services.domain.Settings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class DownloadService {
private final int MAX_POOL_SIZE = 12;
private final SettingsService settingsService;
private final DataService dataService;
private final MetadataService metadataService;
private final List<Host> hosts;
private final Map<Host, AtomicInteger> threadCount = new HashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final List<DownloadJob> running = new ArrayList<>();
private final List<DownloadJob> pending = new ArrayList<>();
private final Thread pollThread;
@Autowired
public DownloadService(
SettingsService settingsService,
DataService dataService,
MetadataService metadataService,
List<Host> hosts) {
this.settingsService = settingsService;
this.dataService = dataService;
this.metadataService = metadataService;
this.hosts = hosts;
pollThread =
new Thread(
() -> {
while (!Thread.interrupted()) {
try {
synchronized (this) {
List<DownloadJob> accepted = new ArrayList<>();
List<DownloadJob> candidates = getCandidates(candidateCount());
candidates.forEach(
c -> {
if (canRun(c.getImage().getHost())) {
accepted.add(c);
}
});
pending.removeAll(accepted);
accepted.forEach(this::schedule);
accepted.clear();
this.wait();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
},
"Download scheduler thread");
}
@PostConstruct
private void init() {
pollThread.start();
}
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
pollThread.interrupt();
executor.shutdown();
dataService
.findAllPosts()
.forEach(
p -> {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
log.warn("Some jobs are still running!, forcing shutdown");
executor.shutdownNow();
}
}
private synchronized void stopRunning(@NonNull String postId) {
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = running.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> postIds) {
stop(
Objects.requireNonNullElseGet(
postIds,
() ->
dataService.findAllPosts().stream()
.map(Post::getPostId)
.collect(Collectors.toList())));
}
public void restartAll(List<String> posIds) {
restart(
Objects.requireNonNullElseGet(
posIds,
() ->
dataService.findAllPosts().stream()
.map(Post::getPostId)
.collect(Collectors.toList())));
}
private void restart(@NonNull List<String> postIds) {
Map<Post, Collection<Image>> data = new HashMap<>();
for (String postId : postIds) {
if (isPending(postId)) {
log.warn(
String.format("Cannot restart, jobs are currently running for post id %s", postIds));
continue;
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
continue;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postIds));
data.put(post, images);
}
enqueue(data);
}
private synchronized boolean isPending(String postId) {
return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private synchronized boolean isRunning(String postId) {
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private synchronized void stop(List<String> postIds) {
for (String postId : postIds) {
final Post post = dataService.findPostByPostId(postId).orElseThrow();
if (post == null) {
continue;
}
pending.removeIf(p -> p.getPost().equals(post));
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
}
metadataService.stopFetchingMetadata(postIds);
}
private boolean canRun(Host host) {
boolean canRun;
int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
canRun =
threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
&& (settingsService.getSettings().getMaxTotalThreads() == 0
? totalRunning < MAX_POOL_SIZE
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
if (canRun) {
threadCount.get(host).incrementAndGet();
return true;
}
return false;
}
private Map<Host, Integer> candidateCount() {
HashMap<Host, Integer> map = new HashMap<>();
hosts.forEach(
h -> {
AtomicInteger count = threadCount.get(h);
if (count == null) {
count = new AtomicInteger(0);
threadCount.put(h, count);
}
map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
});
return map;
}
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount) {
List<DownloadJob> candidates = new ArrayList<>();
for (DownloadJob downloadJob : pending) {
Host host = downloadJob.getImage().getHost();
Integer maxPerHost = candidateCount.get(host);
if (maxPerHost > 0) {
candidates.add(downloadJob);
candidateCount.put(host, maxPerHost - 1);
}
}
return candidates;
}
public void enqueue(Map<Post, Collection<Image>> images) {
synchronized (this) {
for (Map.Entry<Post, Collection<Image>> entry : images.entrySet()) {
entry.getKey().setStatus(Status.PENDING);
dataService.updatePostStatus(entry.getKey().getStatus(), entry.getKey().getId());
for (Image image : entry.getValue()) {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob =
new DownloadJob(
entry.getKey(), image, (Settings) settingsService.getSettings().clone());
pending.add(downloadJob);
}
}
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
this.notify();
}
}
private synchronized void schedule(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new DownloadJobWrapper(downloadJob));
running.add(downloadJob);
}
public void afterJobFinish(DownloadJob downloadJob) {
synchronized (this) {
running.remove(downloadJob);
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
if (!isPending(downloadJob.getPost().getPostId())
&& !isRunning(downloadJob.getPost().getPostId())) {
dataService.finishPost(downloadJob.getPost());
}
this.notify();
}
}
public int pendingCount() {
return pending.size();
}
public int runningCount() {
return running.size();
}
}
@@ -0,0 +1,34 @@
package me.vripper.event;
import lombok.Getter;
@Getter
public class Event<T> {
private final T data;
private final Kind kind;
private Event(Kind kind, T data) {
this.data = data;
this.kind = kind;
}
public static <T> Event<T> wrap(Kind kind, T data) {
return new Event<>(kind, data);
}
public enum Kind {
POST_UPDATE,
POST_REMOVE,
IMAGE_UPDATE,
METADATA_UPDATE,
QUEUED_UPDATE,
QUEUED_REMOVE,
LOG_EVENT_UPDATE,
LOG_EVENT_REMOVE,
VG_USER,
GLOBAL_STATE,
BYTES_PER_SECOND,
SETTINGS_UPDATE
}
}
@@ -0,0 +1,26 @@
package me.vripper.event;
import jakarta.annotation.PreDestroy;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
@Service
public class EventBus {
public static final Sinks.EmitFailureHandler RETRY = (signalType, emitResult) -> true;
private final Sinks.Many<Event<?>> sink = Sinks.many().multicast().onBackpressureBuffer();
public void publishEvent(Event<?> event) {
sink.emitNext(event, RETRY);
}
public Flux<Event<?>> flux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(RETRY);
}
}
@@ -0,0 +1,12 @@
package me.vripper.exception;
public class DownloadException extends Exception {
public DownloadException(String message) {
super(message);
}
public DownloadException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,15 @@
package me.vripper.exception;
public class HostException extends Exception {
public HostException(Throwable e) {
super(e);
}
public HostException(String message) {
super(message);
}
public HostException(String message, Throwable e) {
super(message, e);
}
}
@@ -0,0 +1,8 @@
package me.vripper.exception;
public class HtmlProcessorException extends Exception {
public HtmlProcessorException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,16 @@
package me.vripper.exception;
public class PostParseException extends Exception {
public PostParseException(String message) {
super(message);
}
public PostParseException(String message, Throwable e) {
super(message, e);
}
public PostParseException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,16 @@
package me.vripper.exception;
public class QueueException extends Exception {
public QueueException(String message) {
super(message);
}
public QueueException(String message, Throwable e) {
super(message, e);
}
public QueueException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,10 @@
package me.vripper.exception;
import java.io.IOException;
public class RenameException extends Exception {
public RenameException(String message, IOException e) {
super(message, e);
}
}
@@ -0,0 +1,8 @@
package me.vripper.exception;
public class ValidationException extends Exception {
public ValidationException(String message) {
super(message);
}
}
@@ -0,0 +1,11 @@
package me.vripper.exception;
public class VripperException extends Exception {
public VripperException(String message) {
super(message);
}
public VripperException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,7 @@
package me.vripper.exception;
public class XpathException extends Exception {
public XpathException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,124 @@
package me.vripper.host;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.ConnectionService;
import me.vripper.services.HostService;
import me.vripper.services.HtmlProcessorService;
import me.vripper.services.XpathService;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class AcidimgHost extends Host {
private static final String host = "acidimg.cc";
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final ConnectionService cm;
private final HostService hostService;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
@Autowired
public AcidimgHost(
ConnectionService cm,
HostService hostService,
XpathService xpathService,
HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.hostService = hostService;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node contDiv;
try {
log.debug(
String.format(
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, image.getUrl()));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
log.debug(String.format("Click button found for %s", image.getUrl()));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(image.getUrl(), context);
httpPost.addHeader("Referer", image.getUrl());
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
try {
httpPost.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
throw new HostException(e);
}
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response =
(CloseableHttpResponse) client.execute(httpPost, context)) {
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
throw new HostException(e);
}
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Cannot find the image node");
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,86 @@
package me.vripper.host;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class DPicMeHost extends Host {
private static final String IMG_XPATH = "//img[@id='pic']";
private static final String host = "dpic.me";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public DPicMeHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
HostService.Response response =
hostService.getResponse(image.getUrl().replace("http://", "https://"), context);
Document doc = response.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException(
String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.getUrl()));
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(e -> e.getTextContent().trim())
.orElse("");
String imgUrl =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("src"))
.map(e -> e.getTextContent().trim())
.orElse("");
String defaultName = UUID.randomUUID().toString();
int index = imgUrl.lastIndexOf('/');
if (index != -1 && index < imgUrl.length()) {
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
}
return new HostService.NameUrl(imgTitle.isEmpty() ? defaultName : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,43 @@
package me.vripper.host;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public abstract class Host {
public abstract String getHost();
public abstract String getLookup();
public boolean isSupported(String url) {
return url.contains(getLookup());
}
public abstract HostService.NameUrl getNameAndUrl(
final Image image, final HttpClientContext context) throws HostException;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Host host = (Host) o;
return Objects.equals(getHost(), host.getHost());
}
@Override
public int hashCode() {
return Objects.hash(getHost());
}
@Override
public String toString() {
return getHost();
}
}
@@ -0,0 +1,104 @@
package me.vripper.host;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageBamHost extends Host {
private static final String host = "imagebam.com";
private static final String IMG_XPATH = "//img[contains(@class,'main-image')]";
private static final String CONTINUE_XPATH = "//*[contains(text(), 'Continue')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageBamHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
context.setCookieStore(new BasicCookieStore());
BasicClientCookie cookie = new BasicClientCookie("nsfw_inter", "1");
cookie.setDomain("www.imagebam.com");
context.getCookieStore().addCookie(cookie);
HostService.Response response = hostService.getResponse(image.getUrl(), context);
Document doc = response.getDocument();
try {
log.debug(
String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, image.getUrl()));
if (xpathService.getAsNode(doc, CONTINUE_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
response = hostService.getResponse(image.getUrl(), context);
doc = response.getDocument();
}
} catch (XpathException e) {
throw new HostException(e);
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException(
String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.getUrl()));
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(e -> e.getTextContent().trim())
.orElse("");
String imgUrl =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("src"))
.map(e -> e.getTextContent().trim())
.orElse("");
String defaultName = UUID.randomUUID().toString();
int index = imgUrl.lastIndexOf('/');
if (index != -1 && index < imgUrl.length()) {
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
}
return new HostService.NameUrl(imgTitle.isEmpty() ? defaultName : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageTwistHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "imagetwist.com";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageTwistHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(Node::getTextContent)
.map(String::trim)
.orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,85 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageVenueHost extends Host {
private static final String host = "imagevenue.com";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageVenueHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
resp = hostService.getResponse(url, context);
doc = resp.getDocument();
}
} catch (XpathException e) {
throw new HostException(e);
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,72 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageZillaHost extends Host {
private static final String host = "imagezilla.net";
private static final String lookup = "imagezilla.net/show";
private static final String IMG_XPATH = "//img[@id='photo']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageZillaHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return lookup;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
String title;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
log.debug(String.format("Resolving name for %s", image.getUrl()));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
}
} catch (XpathException e) {
throw new HostException(e);
}
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(image.getUrl());
}
try {
return new HostService.NameUrl(title, image.getUrl().replace("show", "images"));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,68 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImgSpiceHost extends Host {
private static final String host = "imgspice.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,65 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImgboxHost extends Host {
private static final String host = "imgbox.com";
private static final String IMG_XPATH = "//img[@id='img']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgboxHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,40 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class ImxHost extends Host {
private static final String host = "imx.to";
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = String.format("IMG_%04d", image.getIndex() + 1);
String imgUrl =
image
.getThumbUrl()
.replace("http:", "https:")
.replace("upload/small/", "u/i/")
.replace("u/t/", "u/i/");
return new HostService.NameUrl(imgTitle, imgUrl);
}
}
@@ -0,0 +1,76 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PimpandhostHost extends Host {
private static final String host = "pimpandhost.com";
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PimpandhostHost(HostService hostService, XpathService xpathService) {
super();
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url;
try {
url = hostService.appendUri(image.getUrl().replace("http://", "https://"), "size=original");
} catch (Exception e) {
throw new HostException(e);
}
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl =
"https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PixRouteHost extends Host {
private static final String host = "pixroute.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixRouteHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
return new HostService.NameUrl(
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
imgNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,66 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PixhostHost extends Host {
private static final String host = "pixhost.to";
private static final String lookup = "pixhost.to/show";
private static final String IMG_XPATH = "//img[@id='image']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixhostHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return lookup;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,73 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PixxxelsHost extends Host {
private static final String host = "pixxxels.cc";
private static final String IMG_XPATH = "//*[@id='download']";
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixxxelsHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = titleNode.getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,74 @@
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PostImgHost extends Host {
private static final String host = "postimg.cc";
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
private static final String IMG_XPATH = "//a[@id='download']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PostImgHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node urlNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle =
Optional.ofNullable(titleNode)
.map(node -> node.getTextContent().trim())
.orElseGet(() -> hostService.getDefaultImageName(url));
return new HostService.NameUrl(
imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,75 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class TurboImageHost extends Host {
private static final String host = "turboimagehost.com";
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
private static final String IMG_XPATH = "//img[@id='imageid']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public TurboImageHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
String title;
try {
log.debug(
String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.getUrl()));
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Resolving name for %s", image.getUrl()));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
}
} catch (XpathException e) {
throw new HostException(e);
}
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(image.getUrl());
}
try {
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
return new HostService.NameUrl(
title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ViprImHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "vipr.im";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ViprImHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(Node::getTextContent)
.map(String::trim)
.orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,78 @@
package me.vripper.jpa;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import me.vripper.download.DownloadService;
import me.vripper.services.ThreadPoolService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Component
@EnableScheduling
public class Management {
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Pattern BACKUP_FILE_PATTERN =
Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
private final JdbcTemplate jdbcTemplate;
private final String backupFolder;
private final ThreadPoolService threadPoolService;
private final DownloadService downloadService;
public Management(
JdbcTemplate jdbcTemplate,
@Value("${base.dir}") String baseDir,
@Value("${base.dir.name}") String baseDirName,
ThreadPoolService threadPoolService,
DownloadService downloadService) {
this.jdbcTemplate = jdbcTemplate;
this.threadPoolService = threadPoolService;
this.downloadService = downloadService;
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
}
@PreDestroy
public void destroy() {
try {
threadPoolService.destroy();
downloadService.destroy();
this.jdbcTemplate.execute("SHUTDOWN");
} catch (Exception ignored) {
Thread.currentThread().interrupt();
}
}
@PostConstruct
@Scheduled(cron = "0 0 0 ? * *")
private void backup() {
for (File file :
Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[] {})) {
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
if (matcher.find()) {
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
if (localDate.isEqual(LocalDate.now())) {
return;
}
}
}
// create a backup file
File backupFile =
new File(
backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
}
}
@@ -0,0 +1,24 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
public class DateTimeSerializer extends StdSerializer<LocalDateTime> {
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
protected DateTimeSerializer() {
super(LocalDateTime.class);
}
@Override
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
throws IOException {
gen.writeString(value.format(DATE_TIME_FORMATTER));
}
}
@@ -0,0 +1,68 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import me.vripper.host.Host;
import me.vripper.jpa.domain.enums.Status;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Image {
@JsonIgnore protected Long id;
@JsonIgnore private Host host;
private String url;
private String thumbUrl;
private int index;
private long current = 0;
private long total = 0;
private Status status;
private String postId;
@JsonIgnore private Long postIdRef;
public Image(String postId, String url, String thumbUrl, Host host, int index) {
this.postId = postId;
this.url = url;
this.thumbUrl = thumbUrl;
this.host = host;
this.index = index;
status = Status.STOPPED;
}
public void increase(int read) {
current += read;
}
public void init() {
current = 0;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return Objects.equals(url, image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -0,0 +1,51 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.time.LocalDateTime;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class LogEvent {
private Long id;
private Type type;
private Status status;
@JsonSerialize(using = DateTimeSerializer.class)
private LocalDateTime time;
private String message;
public LogEvent(Type type, Status status, LocalDateTime time, String message) {
this.type = type;
this.status = status;
this.time = time;
this.message = message;
}
public enum Type {
POST,
QUEUED,
THANKS,
METADATA,
SCAN,
DOWNLOAD,
// Below are deprecated events
METADATA_CACHE_MISS,
QUEUED_CACHE_MISS
}
public enum Status {
PENDING,
PROCESSING,
DONE,
ERROR
}
}
@@ -0,0 +1,30 @@
package me.vripper.jpa.domain;
import java.util.Collections;
import java.util.List;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@NoArgsConstructor
public class Metadata {
private Long postIdRef;
private String postId;
private String postedBy;
private List<String> resolvedNames = Collections.emptyList();
public static Metadata from(Metadata metadata) {
Metadata copy = new Metadata();
copy.postIdRef = metadata.postIdRef;
copy.postId = metadata.postId;
copy.postedBy = metadata.postedBy;
copy.resolvedNames = List.copyOf(metadata.resolvedNames);
return copy;
}
}
@@ -0,0 +1,89 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import me.vripper.jpa.domain.enums.Status;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Post {
@JsonIgnore private Long id;
private Status status;
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private int done;
private int total;
private Set<String> hosts;
private String forum;
@JsonIgnore private String securityToken;
@JsonIgnore private String downloadDirectory;
private boolean thanked;
private Set<String> previews = Collections.emptySet();
private Metadata metadata;
private int rank = Integer.MAX_VALUE;
@JsonSerialize(using = DateTimeSerializer.class)
private LocalDateTime addedOn;
public Post(
String title,
String url,
String postId,
String threadId,
String threadTitle,
String forum,
String securityToken) {
this.title = title;
this.url = url;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
addedOn = LocalDateTime.now();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return Objects.equals(postId, post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -0,0 +1,55 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class Queued {
@JsonIgnore private Long id;
private String link;
private String threadId;
private String postId;
private int total = 0;
private boolean loading = true;
public Queued(String link, String threadId, String postId) {
this();
this.link = link;
this.threadId = threadId;
this.postId = postId;
}
public void done() {
this.loading = false;
}
public void increment() {
this.total++;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Queued that = (Queued) o;
return Objects.equals(threadId, that.threadId);
}
@Override
public int hashCode() {
return Objects.hash(threadId);
}
}
@@ -0,0 +1,10 @@
package me.vripper.jpa.domain.enums;
public enum Status {
PENDING,
DOWNLOADING,
COMPLETE,
ERROR,
STOPPED,
PARTIAL
}
@@ -0,0 +1,31 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.enums.Status;
public interface IImageRepository extends IRepository {
Image save(Image image);
void deleteAllByPostId(String postId);
List<Image> findByPostId(String postId);
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
int stopByPostIdAndIsNotCompleted(String postId);
List<Image> findByPostIdAndIsError(String postId);
Optional<Image> findById(Long id);
int updateStatus(Status status, Long id);
int updateCurrent(long current, Long id);
int updateTotal(long total, Long id);
}
@@ -0,0 +1,20 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.LogEvent;
public interface ILogEventRepository extends IRepository {
LogEvent save(LogEvent logEvent);
LogEvent update(LogEvent logEvent);
List<LogEvent> findAll();
Optional<LogEvent> findById(Long id);
void delete(Long id);
void deleteAll();
}
@@ -0,0 +1,13 @@
package me.vripper.jpa.repositories;
import java.util.Optional;
import me.vripper.jpa.domain.Metadata;
public interface IMetadataRepository extends IRepository {
Metadata save(Metadata metadata);
Optional<Metadata> findByPostId(String postId);
int deleteByPostId(String postId);
}
@@ -0,0 +1,37 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
public interface IPostRepository extends IRepository {
Post save(Post post);
Optional<Post> findByPostId(String postId);
Optional<Post> findById(Long id);
List<String> findCompleted();
List<Post> findAll();
boolean existByPostId(String postId);
int setDownloadingToStopped();
int deleteByPostId(String postId);
int updateStatus(Status status, Long id);
int updateDone(int done, Long id);
int updateFolderName(String postFolderName, Long id);
int updateTitle(String title, Long id);
int updateThanked(boolean thanked, Long id);
int updateRank(int rank, Long id);
}
@@ -0,0 +1,19 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Queued;
public interface IQueuedRepository extends IRepository {
Queued save(Queued queued);
Optional<Queued> findByThreadId(String threadId);
List<Queued> findAll();
Optional<Queued> findById(Long id);
int deleteByThreadId(String threadId);
void deleteAll();
}
@@ -0,0 +1,3 @@
package me.vripper.jpa.repositories;
public interface IRepository {}
@@ -0,0 +1,157 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.SpringContext;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IImageRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class ImageRepository implements IImageRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public ImageRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_IMAGE", Long.class);
}
@Override
public Image save(Image image) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, THUMB_URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?,?)",
id,
image.getCurrent(),
image.getHost().getHost(),
image.getIndex(),
image.getPostId(),
image.getStatus().name(),
image.getTotal(),
image.getUrl(),
image.getThumbUrl(),
image.getPostIdRef());
image.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return image;
}
@Override
public void deleteAllByPostId(String postId) {
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
}
@Override
public List<Image> findByPostId(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE WHERE POST_ID = ?", new ImageRowMapper(), postId);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'", Integer.class);
}
@Override
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
new ImageRowMapper(),
postId);
}
@Override
public int stopByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
postId);
}
@Override
public List<Image> findByPostIdAndIsError(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
new ImageRowMapper(),
postId);
}
@Override
public Optional<Image> findById(Long id) {
List<Image> images =
jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.ID = ?", new ImageRowMapper(), id);
if (images.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(images.get(0));
}
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?", status.name(), id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return mutationCount;
}
@Override
public int updateCurrent(long current, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?", current, id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return mutationCount;
}
@Override
public int updateTotal(long total, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?", total, id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return mutationCount;
}
}
class ImageRowMapper implements RowMapper<Image> {
@Override
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
Image image = new Image();
image.setId(rs.getLong("ID"));
String host = rs.getString("HOST");
image.setHost(
SpringContext.getBeansOfType(Host.class).values().stream()
.filter(e -> e.getHost().equals(host))
.findAny()
.orElse(null));
image.setUrl(rs.getString("URL"));
image.setThumbUrl(rs.getString("THUMB_URL"));
image.setIndex(rs.getInt("INDEX"));
image.setCurrent(rs.getLong("CURRENT"));
image.setTotal(rs.getLong("TOTAL"));
image.setStatus(Status.valueOf(rs.getString("STATUS")));
image.setPostId(rs.getString("POST_ID"));
image.setPostIdRef(rs.getLong("POST_ID_REF"));
return image;
}
}
@@ -0,0 +1,124 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.SettingsService;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class LogEventRepository implements ILogEventRepository {
private final JdbcTemplate jdbcTemplate;
private final SettingsService settingsService;
private final EventBus eventBus;
public LogEventRepository(
JdbcTemplate jdbcTemplate, SettingsService settingsService, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.settingsService = settingsService;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_EVENT", Long.class);
}
@Override
public synchronized LogEvent save(@NonNull LogEvent logEvent) {
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
if (count > maxRecords) {
List<Long> idList =
jdbcTemplate.queryForList(
"SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
idList.forEach(this::delete);
}
long id = nextId();
jdbcTemplate.update(
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
id,
logEvent.getType().name(),
logEvent.getStatus().name(),
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
logEvent.getMessage());
logEvent.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, id));
return logEvent;
}
@Override
public LogEvent update(@NonNull LogEvent logEvent) {
if (logEvent.getId() == null) {
log.warn("Cannot update entity with null id");
return logEvent;
}
jdbcTemplate.update(
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
logEvent.getType().name(),
logEvent.getStatus().name(),
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
logEvent.getMessage(),
logEvent.getId());
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, logEvent.getId()));
return logEvent;
}
@Override
public Optional<LogEvent> findById(Long id) {
List<LogEvent> logEvents =
jdbcTemplate.query("SELECT * FROM EVENT WHERE ID = ?", new LogEventRowMapper(), id);
if (logEvents.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(logEvents.get(0));
}
}
@Override
public List<LogEvent> findAll() {
return jdbcTemplate.query("SELECT * FROM EVENT", new LogEventRowMapper());
}
@Override
public void delete(Long id) {
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_REMOVE, id));
}
@Override
public void deleteAll() {
jdbcTemplate.update("DELETE FROM EVENT");
}
}
class LogEventRowMapper implements RowMapper<LogEvent> {
@Override
public LogEvent mapRow(ResultSet rs, int rowNum) throws SQLException {
LogEvent logEvent = new LogEvent();
logEvent.setId(rs.getLong("ID"));
logEvent.setType(LogEvent.Type.valueOf(rs.getString("TYPE")));
logEvent.setStatus(LogEvent.Status.valueOf(rs.getString("STATUS")));
logEvent.setTime(
LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
logEvent.setMessage(rs.getString("MESSAGE"));
return logEvent;
}
}
@@ -0,0 +1,75 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.repositories.IMetadataRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class MetadataRepository implements IMetadataRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public MetadataRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
@Override
public Metadata save(Metadata metadata) {
jdbcTemplate.update(
"INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
metadata.getPostIdRef(),
metadata.getPostId(),
metadata.getPostedBy(),
String.join("%sep%", metadata.getResolvedNames()));
eventBus.publishEvent(Event.wrap(Event.Kind.METADATA_UPDATE, metadata.getPostIdRef()));
return metadata;
}
@Override
public Optional<Metadata> findByPostId(String postId) {
List<Metadata> metadata =
jdbcTemplate.query(
"SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
new MetadataRowMapper(),
postId);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?", postId);
}
}
class MetadataRowMapper implements RowMapper<Metadata> {
@Override
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
Metadata metadata = new Metadata();
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
metadata.setPostId(rs.getString("POST_ID"));
metadata.setPostedBy(rs.getString("POSTED_BY"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
return metadata;
}
}
@@ -0,0 +1,229 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IPostRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class PostRepository implements IPostRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public PostRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_POST", Long.class);
}
@Override
public Post save(Post post) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON, RANK) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
String.join(";", post.getHosts()),
post.getDownloadDirectory(),
post.getPostId(),
String.join(";", post.getPreviews()),
post.getSecurityToken(),
post.getStatus().name(),
post.isThanked(),
post.getThreadId(),
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
post.getUrl(),
Timestamp.valueOf(post.getAddedOn()),
post.getRank());
post.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return post;
}
@Override
public Optional<Post> findByPostId(String postId) {
List<Post> posts =
jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
new PostRowMapper(),
postId);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<String> findCompleted() {
return jdbcTemplate.query(
"SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total",
((rs, rowNum) -> rs.getString("POST_ID")));
}
@Override
public Optional<Post> findById(Long id) {
List<Post> posts =
jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
new PostRowMapper(),
id);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<Post> findAll() {
return jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
new PostRowMapper());
}
@Override
public boolean existByPostId(String postId) {
Integer count =
jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?", Integer.class, postId);
if (count == null) {
return false;
} else {
return count > 0;
}
}
@Override
public int setDownloadingToStopped() {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'");
}
@Override
public int deleteByPostId(String postId) {
int mutationCount =
jdbcTemplate.update("DELETE FROM POST AS post WHERE post.POST_ID = ?", postId);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_REMOVE, postId));
return mutationCount;
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?", status.name(), id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateDone(int done, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?", done, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateFolderName(String postFolderName, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
postFolderName,
id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateTitle(String title, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?", title, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateThanked(boolean thanked, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?", thanked, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateRank(int rank, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.RANK = ? WHERE post.ID = ?", rank, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
}
class PostRowMapper implements RowMapper<Post> {
private static final String DELIMITER = ";";
@Override
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
Post post = new Post();
post.setId(rs.getLong("post.ID"));
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
post.setPostId(rs.getString("post.POST_ID"));
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
post.setThreadId(rs.getString("post.THREAD_ID"));
post.setTitle(rs.getString("post.TITLE"));
post.setUrl(rs.getString("post.URL"));
post.setDone(rs.getInt("post.DONE"));
post.setTotal(rs.getInt("post.TOTAL"));
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
post.setForum(rs.getString("post.FORUM"));
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
post.setThanked(rs.getBoolean("post.THANKED"));
String previews;
if ((previews = rs.getString("post.PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
post.setRank(rs.getInt("post.RANK"));
Long metadataId = rs.getLong("metadata.POST_ID_REF");
if (!rs.wasNull()) {
Metadata metadata = new Metadata();
metadata.setPostIdRef(metadataId);
metadata.setPostId(rs.getString("metadata.POST_ID"));
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
post.setMetadata(metadata);
}
return post;
}
}
@@ -0,0 +1,106 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.IQueuedRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class QueuedRepository implements IQueuedRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public QueuedRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_QUEUED", Long.class);
}
@Override
public Queued save(Queued queued) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
id,
queued.getTotal(),
queued.getLink(),
queued.isLoading(),
queued.getPostId(),
queued.getThreadId());
queued.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_UPDATE, id));
return queued;
}
@Override
public Optional<Queued> findByThreadId(String threadId) {
List<Queued> queuedList =
jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
new QueuedRowMapper(),
threadId);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public List<Queued> findAll() {
return jdbcTemplate.query("SELECT * FROM QUEUED", new QueuedRowMapper());
}
@Override
public Optional<Queued> findById(Long id) {
List<Queued> queuedList =
jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?", new QueuedRowMapper(), id);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public int deleteByThreadId(String threadId) {
int mutationCount =
jdbcTemplate.update("DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?", threadId);
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_REMOVE, threadId));
return mutationCount;
}
@Override
public void deleteAll() {
jdbcTemplate.update("DELETE FROM QUEUED");
}
}
class QueuedRowMapper implements RowMapper<Queued> {
@Override
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
Queued queued = new Queued();
queued.setId(rs.getLong("ID"));
queued.setLink(rs.getString("LINK"));
queued.setThreadId(rs.getString("THREAD_ID"));
queued.setPostId(rs.getString("POST_ID"));
queued.setTotal(rs.getInt("TOTAL"));
queued.setLoading(rs.getBoolean("LOADING"));
return queued;
}
}
@@ -0,0 +1,153 @@
package me.vripper.services;
import jakarta.annotation.PreDestroy;
import java.net.URI;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.download.DownloadJob;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.services.domain.Settings;
import net.jodah.failsafe.RetryPolicy;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@EnableScheduling
@Slf4j
public class ConnectionService {
private final Disposable disposable;
private PoolingHttpClientConnectionManager pcm;
private RequestConfig rc;
@Getter private RetryPolicy<Object> retryPolicy;
private int connectionTimeout;
private int maxAttempts;
public ConnectionService(EventBus eventBus, SettingsService settingsService) {
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
maxAttempts = settingsService.getSettings().getMaxAttempts();
disposable =
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.map(e -> ((Settings) e.getData()))
.subscribe(
settings -> {
if (connectionTimeout != settings.getConnectionTimeout()) {
connectionTimeout = settings.getConnectionTimeout();
buildRequestConfig();
}
if (maxAttempts != settings.getMaxAttempts()) {
maxAttempts = settings.getMaxAttempts();
buildRetryPolicy();
}
});
buildRequestConfig();
buildRetryPolicy();
buildConnectionPool();
}
@PreDestroy
private void destroy() {
disposable.dispose();
}
private void buildRetryPolicy() {
retryPolicy =
new RetryPolicy<>()
.withDelay(2, 5, ChronoUnit.SECONDS)
.withMaxAttempts(maxAttempts)
.onFailedAttempt(
e ->
log.warn(
String.format("#%d tries failed", e.getAttemptCount()),
e.getLastFailure()));
}
private void buildConnectionPool() {
pcm = new PoolingHttpClientConnectionManager();
pcm.setMaxTotal(Integer.MAX_VALUE);
pcm.setDefaultMaxPerRoute(Integer.MAX_VALUE);
}
private void buildRequestConfig() {
rc =
RequestConfig.custom()
.setConnectionRequestTimeout(connectionTimeout * 1000)
.setConnectTimeout(connectionTimeout * 1000)
.setSocketTimeout(connectionTimeout * 1000)
.setCookieSpec(CookieSpecs.STANDARD)
.build();
}
@Scheduled(fixedDelay = 5_000)
private void idleConnectionMonitoring() {
pcm.closeExpiredConnections();
pcm.closeIdleConnections(30, TimeUnit.SECONDS);
}
public HttpClientBuilder getClient() {
return HttpClients.custom()
.setConnectionManager(pcm)
.setRedirectStrategy(new LaxRedirectStrategy())
.disableAutomaticRetries()
.setDefaultRequestConfig(rc);
}
public HttpGet buildHttpGet(String url, final HttpClientContext context) {
HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
httpGet.addHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpGet);
return httpGet;
}
public HttpPost buildHttpPost(String url, final HttpClientContext context) {
HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
httpPost.addHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpPost);
return httpPost;
}
public HttpGet buildHttpGet(URI uri, final HttpClientContext context) {
HttpGet httpGet = new HttpGet(uri);
httpGet.addHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpGet);
return httpGet;
}
public void addToContext(HttpClientContext context, AbstractExecutionAwareRequest request) {
if (context != null) {
List<AbstractExecutionAwareRequest> requests =
(List<AbstractExecutionAwareRequest>)
context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
requests.add(request);
}
}
}
}
@@ -0,0 +1,242 @@
package me.vripper.services;
import java.util.*;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.domain.*;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IImageRepository;
import me.vripper.jpa.repositories.IMetadataRepository;
import me.vripper.jpa.repositories.IPostRepository;
import me.vripper.jpa.repositories.IQueuedRepository;
import me.vripper.jpa.repositories.impl.LogEventRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@Transactional
@Slf4j
public class DataService {
private final IPostRepository postRepository;
private final IImageRepository imageRepository;
private final IQueuedRepository queuedRepository;
private final IMetadataRepository metadataRepository;
private final SettingsService settingsService;
private final LogEventRepository eventRepository;
@Autowired
public DataService(
IPostRepository postRepository,
IImageRepository imageRepository,
IQueuedRepository queuedRepository,
IMetadataRepository metadataRepository,
SettingsService settingsService,
LogEventRepository eventRepository) {
this.postRepository = postRepository;
this.imageRepository = imageRepository;
this.queuedRepository = queuedRepository;
this.metadataRepository = metadataRepository;
this.settingsService = settingsService;
this.eventRepository = eventRepository;
}
private void save(Post post) {
postRepository.save(post);
}
private void save(Queued queued) {
queuedRepository.save(queued);
}
private void save(Image image) {
imageRepository.save(image);
}
public boolean exists(String postId) {
return postRepository.existByPostId(postId);
}
public void newPost(Post post, Collection<Image> images) {
save(post);
images.forEach(
image -> {
image.setPostIdRef(post.getId());
save(image);
});
sortPostsByRank();
}
public synchronized void afterJobFinish(Image image, Post post) {
if (image.getStatus().equals(Status.COMPLETE)) {
post.setDone(post.getDone() + 1);
updatePostDone(post.getDone(), post.getId());
} else if (image.getStatus().equals(Status.ERROR)) {
post.setStatus(Status.PARTIAL);
updatePostStatus(post.getStatus(), post.getId());
}
}
private void updatePostDone(int done, Long id) {
postRepository.updateDone(done, id);
}
public void updatePostStatus(Status status, Long id) {
postRepository.updateStatus(status, id);
}
public void updatePostThanks(boolean thanked, Long id) {
postRepository.updateThanked(thanked, id);
}
public void updatePostTitle(String title, Long id) {
postRepository.updateTitle(title, id);
}
public void updatePostDownloadDirectory(String postFolderName, Long id) {
postRepository.updateFolderName(postFolderName, id);
}
public void updatePostRank(int rank, Long id) {
postRepository.updateRank(rank, id);
}
public void finishPost(@NonNull Post post) {
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
post.setStatus(Status.ERROR);
updatePostStatus(post.getStatus(), post.getId());
} else {
if (post.getDone() < post.getTotal()) {
post.setStatus(Status.STOPPED);
updatePostStatus(post.getStatus(), post.getId());
} else {
post.setStatus(Status.COMPLETE);
updatePostStatus(post.getStatus(), post.getId());
if (settingsService.getSettings().getClearCompleted()) {
remove(List.of(post.getPostId()));
}
}
}
}
private void remove(@NonNull final List<String> postIds) {
for (String postId : postIds) {
imageRepository.deleteAllByPostId(postId);
metadataRepository.deleteByPostId(postId);
postRepository.deleteByPostId(postId);
}
sortPostsByRank();
}
public void newQueueLink(@NonNull final Queued queued) {
save(queued);
}
public void removeQueueLink(@NonNull final String threadId) {
queuedRepository.deleteByThreadId(threadId);
}
public List<String> clearCompleted() {
List<String> completed = postRepository.findCompleted();
remove(completed);
return completed;
}
public void removeAll(final List<String> postIds) {
remove(
Objects.requireNonNullElse(
postIds,
postRepository.findAll().stream().map(Post::getPostId).collect(Collectors.toList())));
}
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
return imageRepository.findByPostIdAndIsNotCompleted(postId);
}
public long countErrorImages() {
return imageRepository.countError();
}
public List<Image> findImagesByPostId(String postId) {
return imageRepository.findByPostId(postId);
}
public List<Post> findAllPosts() {
return postRepository.findAll();
}
public Optional<Post> findPostByPostId(String postId) {
return postRepository.findByPostId(postId);
}
public void stopImagesByPostIdAndIsNotCompleted(String postId) {
imageRepository.stopByPostIdAndIsNotCompleted(postId);
}
public Optional<Queued> findQueuedByThreadId(String threadId) {
return queuedRepository.findByThreadId(threadId);
}
public List<Queued> findAllQueued() {
return queuedRepository.findAll();
}
public Optional<Post> findById(Long aLong) {
return postRepository.findById(aLong);
}
public Optional<Image> findImageById(Long aLong) {
return imageRepository.findById(aLong);
}
public Optional<Queued> findQueuedById(Long aLong) {
return queuedRepository.findById(aLong);
}
public synchronized void setMetadata(Post post, Metadata metadata) {
if (metadataRepository.findByPostId(post.getPostId()).isEmpty()) {
metadata.setPostIdRef(post.getId());
metadataRepository.save(metadata);
}
}
public Optional<Metadata> findMetadataByPostId(String postId) {
return metadataRepository.findByPostId(postId);
}
public void updateImageStatus(Status status, Long id) {
imageRepository.updateStatus(status, id);
}
public void updateImageCurrent(long current, Long id) {
imageRepository.updateCurrent(current, id);
}
public void updateImageTotal(long total, Long id) {
imageRepository.updateTotal(total, id);
}
public Optional<LogEvent> findEventById(Long id) {
return eventRepository.findById(id);
}
public List<LogEvent> findAllEvents() {
return eventRepository.findAll();
}
public void clearQueueLinks() {
queuedRepository.deleteAll();
}
public synchronized void sortPostsByRank() {
List<Post> posts = findAllPosts();
posts.sort(Comparator.comparing(Post::getAddedOn));
for (int i = 0; i < posts.size(); i++) {
posts.get(i).setRank(i);
updatePostRank(i, posts.get(i).getId());
}
}
}
@@ -0,0 +1,40 @@
package me.vripper.services;
import java.util.concurrent.atomic.AtomicLong;
import lombok.Getter;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Service
@EnableScheduling
public class DownloadSpeedService {
private final AtomicLong read = new AtomicLong(0);
private final EventBus eventBus;
@Getter private long currentValue;
private boolean allowWrite = false;
public DownloadSpeedService(EventBus eventBus) {
this.eventBus = eventBus;
}
public void increase(long read) {
if (allowWrite) {
this.read.addAndGet(read);
}
}
@Scheduled(fixedDelay = 1000)
private void calc() {
allowWrite = false;
long newValue = read.getAndSet(0);
if (newValue != currentValue) {
currentValue = newValue;
eventBus.publishEvent(Event.wrap(Event.Kind.BYTES_PER_SECOND, currentValue));
}
allowWrite = true;
}
}
@@ -0,0 +1,42 @@
package me.vripper.services;
import lombok.Getter;
import me.vripper.download.DownloadService;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.services.domain.GlobalState;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Service
@EnableScheduling
public class GlobalStateService {
private final DownloadService downloadService;
private final DataService dataService;
private final EventBus eventBus;
@Getter private GlobalState currentState;
@Autowired
public GlobalStateService(
DownloadService downloadService, DataService dataService, EventBus eventBus) {
this.downloadService = downloadService;
this.dataService = dataService;
this.eventBus = eventBus;
}
@Scheduled(fixedDelay = 3000)
private void interval() {
GlobalState newGlobalState =
new GlobalState(
downloadService.runningCount(),
downloadService.pendingCount(),
dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
}
}
}
@@ -0,0 +1,112 @@
package me.vripper.services;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.HtmlProcessorException;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@Service
@Slf4j
public class HostService {
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
@Autowired
public HostService(ConnectionService cm, HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
}
public Response getResponse(final String url, final HttpClientContext context)
throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url, context);
Header[] headers;
log.debug(String.format("Requesting %s", url));
try (CloseableHttpResponse response =
(CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(
String.format(
"Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
log.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
public String appendUri(String uri, String appendQuery) throws URISyntaxException {
URI oldUri = new URI(uri);
String newQuery = oldUri.getQuery();
if (newQuery == null) {
newQuery = appendQuery;
} else {
newQuery += "&" + appendQuery;
}
return new URI(
oldUri.getScheme(),
oldUri.getAuthority(),
oldUri.getPath(),
newQuery,
oldUri.getFragment())
.toString();
}
public String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
@Getter
public static class Response {
private final Document document;
private final Header[] headers;
protected Response(Document document, Header[] headers) {
this.document = document;
this.headers = headers;
}
}
@Getter
public static class NameUrl {
private final String name;
private final String url;
public NameUrl(String name, String url) {
this.name = name;
this.url = url;
}
}
}
@@ -0,0 +1,22 @@
package me.vripper.services;
import me.vripper.exception.HtmlProcessorException;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@Service
public class HtmlProcessorService {
public Document clean(String htmlContent) throws HtmlProcessorException {
try {
TagNode clean = new HtmlCleaner().clean(htmlContent);
return new DomSerializer(new CleanerProperties()).createDOM(clean);
} catch (Exception e) {
throw new HtmlProcessorException(e);
}
}
}
@@ -0,0 +1,49 @@
package me.vripper.services;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.domain.Post;
import me.vripper.tasks.MetadataRunnable;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class MetadataService {
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
private final ThreadPoolService threadPoolService;
public MetadataService(ThreadPoolService threadPoolService) {
this.threadPoolService = threadPoolService;
}
public void startFetchingMetadata(Post post) {
MetadataRunnable runnable = new MetadataRunnable(post);
threadPoolService.getGeneralExecutor().submit(runnable);
fetchingMetadata.put(post.getPostId(), runnable);
}
public void stopFetchingMetadata(List<String> postIds) {
List<MetadataRunnable> stopping = new ArrayList<>();
for (Map.Entry<String, MetadataRunnable> entry : this.fetchingMetadata.entrySet()) {
if (postIds.contains(entry.getKey())) {
stopping.add(entry.getValue());
entry.getValue().stop();
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(MetadataRunnable::isFinished);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
postIds.forEach(fetchingMetadata::remove);
}
}
@@ -0,0 +1,125 @@
package me.vripper.services;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.concurrent.locks.ReentrantLock;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.RenameException;
import me.vripper.jpa.domain.Post;
import me.vripper.services.domain.Settings;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class PathService {
private final DataService dataService;
@Getter private final ReentrantLock directoryAccess = new ReentrantLock();
public PathService(DataService dataService) {
this.dataService = dataService;
}
public final File calcDownloadDirectory(Post post, Settings settings) {
return new File(settings.getDownloadPath(), post.getDownloadDirectory());
}
private File getRootFolder(
@NonNull String forum, @NonNull String threadTitle, Settings settings) {
File sourceFolder =
settings.getSubLocation()
? new File(settings.getDownloadPath(), sanitize(forum))
: new File(settings.getDownloadPath());
return settings.getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
}
public final void createDefaultPostFolder(Post post, Settings settings) {
File downloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
downloadDirectory =
new File(
downloadDirectory,
settings.getAppendPostId()
? sanitize(post.getTitle()) + "_" + post.getPostId()
: sanitize(post.getTitle()));
downloadDirectory = makeDir(downloadDirectory);
post.setDownloadDirectory(
downloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
}
public final void rename(@NonNull String postId, @NonNull String altName, Settings settings)
throws RenameException {
Post post = dataService.findPostByPostId(postId).orElseThrow();
if (altName.equals(post.getTitle())) {
return;
}
// Download have not started yet
if (post.getDownloadDirectory() == null) {
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
return;
}
File newDownloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
newDownloadDirectory = new File(newDownloadDirectory, sanitize(altName));
File currentDownloadDirectory = calcDownloadDirectory(post, settings);
try {
directoryAccess.lock();
Files.move(
currentDownloadDirectory.toPath(),
newDownloadDirectory.toPath(),
StandardCopyOption.ATOMIC_MOVE);
post.setDownloadDirectory(
newDownloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
} catch (IOException e) {
throw new RenameException(
String.format(
"Failed to move files from %s to %s", currentDownloadDirectory, newDownloadDirectory),
e);
} finally {
directoryAccess.unlock();
}
}
private File makeDir(@NonNull final File sourceFolder) {
int counter = 1;
File folder = sourceFolder;
while (folder.exists()) {
folder = new File(sourceFolder.toString() + '.' + counter++);
}
if (!folder.mkdirs()) {
throw new RuntimeException(String.format("Failed to create the folder %s", sourceFolder));
}
return folder;
}
private String sanitize(final String folderName) {
String sanitizedFolderName =
folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
log.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
return sanitizedFolderName;
}
/**
* Will sanitize the image name and remove extension
*
* @param imageName path string
* @return Sanitized local path string
*/
public final String formatImageFileName(String imageName) {
return sanitize(imageName);
}
}
@@ -0,0 +1,72 @@
package me.vripper.services;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import jakarta.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.domain.MultiPostScanParser;
import me.vripper.services.domain.MultiPostScanResult;
import me.vripper.tasks.AddPostRunnable;
import me.vripper.tasks.AddQueuedRunnable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@Slf4j
public class PostService {
private final DataService dataService;
private final ThreadPoolService threadPoolService;
private final LoadingCache<Queued, MultiPostScanResult> cache;
private final Disposable disposable;
@Autowired
public PostService(
DataService dataService, ThreadPoolService threadPoolService, EventBus eventBus) {
this.dataService = dataService;
this.threadPoolService = threadPoolService;
cache =
Caffeine.newBuilder()
.expireAfterWrite(5, TimeUnit.MINUTES)
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
disposable =
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> this.cache.invalidateAll());
}
@PreDestroy
private void destroy() {
if (disposable != null) {
disposable.dispose();
}
}
public void processMultiPost(List<Queued> queuedList) {
for (Queued queued : queuedList) {
if (queued.getPostId() != null) {
threadPoolService
.getGeneralExecutor()
.submit(new AddPostRunnable(queued.getPostId(), queued.getThreadId()));
} else {
threadPoolService.getGeneralExecutor().submit(new AddQueuedRunnable(queued));
}
}
}
public MultiPostScanResult get(Queued queued) throws ExecutionException {
return cache.get(queued);
}
public void remove(String threadId) {
dataService.removeQueueLink(threadId);
}
}
@@ -0,0 +1,304 @@
package me.vripper.services;
import static java.nio.file.StandardOpenOption.*;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.exception.ValidationException;
import me.vripper.services.domain.Settings;
import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
@Service
@Setter
@Slf4j
public class SettingsService {
private final Path configPath;
private final Path customProxiesPath;
private final ObjectMapper om = new ObjectMapper();
private final Set<String> proxies = new HashSet<>();
private final EventBus eventBus;
@Getter private Settings settings = new Settings();
@Value("classpath:proxies.json")
private Resource defaultProxies;
public SettingsService(
@Value("${base.dir}") String baseDir,
@Value("${base.dir.name}") String baseDirName,
EventBus eventBus) {
this.eventBus = eventBus;
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
@PostConstruct
private void init() {
loadViperProxies();
restore();
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
private void loadViperProxies() {
try (InputStream defaultProxiesIs = defaultProxies.getInputStream()) {
List<String> defaultProxies = om.readValue(defaultProxiesIs, new TypeReference<>() {});
List<String> customProxies = new ArrayList<>();
if (customProxiesPath.toFile().exists() && customProxiesPath.toFile().isFile()) {
customProxies = om.readValue(customProxiesPath.toFile(), new TypeReference<>() {});
} else {
if (!customProxiesPath.toFile().createNewFile()) {
log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath());
} else {
try (FileWriter fw = new FileWriter(customProxiesPath.toFile())) {
fw.append("[]").append(System.lineSeparator());
fw.flush();
} catch (IOException e) {
log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath(), e);
}
}
}
proxies.addAll(customProxies);
proxies.addAll(defaultProxies);
} catch (IOException e) {
log.error("Failed to load vipergirls proxies list", e);
}
proxies.add("https://vipergirls.to");
}
public List<String> getProxies() {
return new ArrayList<>(proxies);
}
public void newSettings(Settings settings) {
if (settings.getVLogin() != null && settings.getVLogin()) {
if (!this.settings.getVPassword().equals(settings.getVPassword())) {
settings.setVPassword(DigestUtils.md5Hex(settings.getVPassword()));
}
} else {
settings.setVUsername("");
settings.setVPassword("");
settings.setVThanks(false);
settings.setVLogin(false);
}
this.settings = settings;
save();
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
public void restore() {
try {
if (configPath.toFile().exists()) {
settings = om.readValue(configPath.toFile(), Settings.class);
}
} catch (IOException e) {
log.error("Failed restore user settings", e);
settings = new Settings();
}
if (settings.getDownloadPath() == null) {
settings.setDownloadPath(System.getProperty("user.home"));
}
if (settings.getMaxThreads() == null) {
settings.setMaxThreads(4);
}
if (settings.getMaxTotalThreads() == null) {
settings.setMaxTotalThreads(0);
}
if (settings.getAutoStart() == null) {
settings.setAutoStart(true);
}
if (settings.getVLogin() == null) {
settings.setVLogin(false);
}
if (settings.getVUsername() == null) {
settings.setVUsername("");
}
if (settings.getVPassword() == null) {
settings.setVPassword("");
}
if (settings.getVThanks() == null) {
settings.setVThanks(false);
}
if (settings.getDesktopClipboard() == null) {
settings.setDesktopClipboard(false);
}
if (settings.getForceOrder() == null) {
settings.setForceOrder(false);
}
if (settings.getSubLocation() == null) {
settings.setSubLocation(false);
}
if (settings.getThreadSubLocation() == null) {
settings.setThreadSubLocation(false);
}
if (settings.getClearCompleted() == null) {
settings.setClearCompleted(false);
}
if (settings.getDarkTheme() == null) {
settings.setDarkTheme(false);
}
if (settings.getAppendPostId() == null) {
settings.setAppendPostId(true);
}
if (settings.getLeaveThanksOnStart() == null) {
settings.setLeaveThanksOnStart(false);
}
if (settings.getConnectionTimeout() == null) {
settings.setConnectionTimeout(30);
}
if (settings.getMaxAttempts() == null) {
settings.setMaxAttempts(5);
}
if (settings.getVProxy() == null || !proxies.contains(settings.getVProxy())) {
settings.setVProxy("https://vipergirls.to");
}
if (settings.getMaxEventLog() == null) {
settings.setMaxEventLog(1000);
}
try {
check(this.settings);
} catch (ValidationException e) {
log.error(
String.format("Your settings are invalid, either remove %s, or fix it", configPath), e);
SpringContext.close();
}
save();
}
public void save() {
try {
Files.write(
configPath, om.writeValueAsBytes(settings), CREATE, WRITE, TRUNCATE_EXISTING, SYNC);
} catch (IOException e) {
log.error("Failed to store user settings", e);
}
}
@PreDestroy
private void destroy() {
save();
}
public void check(Settings settings) throws ValidationException {
Path path;
try {
path = Paths.get(settings.getDownloadPath());
} catch (InvalidPathException e) {
throw new ValidationException(String.format("%s is invalid", settings.getDownloadPath()));
}
if (!Files.exists(path)) {
throw new ValidationException(String.format("%s does not exist", settings.getDownloadPath()));
} else if (!Files.isDirectory(path)) {
throw new ValidationException(
String.format("%s is not a directory", settings.getDownloadPath()));
}
if (settings.getMaxTotalThreads() < 0 || settings.getMaxTotalThreads() > 12) {
throw new ValidationException(
String.format(
"Invalid max global concurrent download settings, values must be in [%d,%d]", 0, 12));
}
if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
throw new ValidationException(
String.format(
"Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
}
if (settings.getConnectionTimeout() < 1 || settings.getConnectionTimeout() > 300) {
throw new ValidationException(
String.format("Invalid connection timeout settings, values must be in [%d,%d]", 1, 300));
}
if (settings.getMaxAttempts() < 1 || settings.getMaxAttempts() > 10) {
throw new ValidationException(
String.format("Invalid maximum attempts settings, values must be in [%d,%d]", 1, 10));
}
if (settings.getMaxEventLog() < 100 || settings.getMaxEventLog() > 10_000) {
throw new ValidationException(
String.format(
"Invalid maximum event log record settings, values must be in [%d,%d]", 100, 10_000));
}
if (!settings.getVThanks()
&& settings.getLeaveThanksOnStart() != null
&& settings.getLeaveThanksOnStart()) {
throw new ValidationException("Invalid value, leave thanks must be checked");
}
}
public Theme getTheme() {
return new Theme(this.settings.getDarkTheme());
}
public void setTheme(Theme theme) {
this.settings.setDarkTheme(theme.darkTheme);
save();
}
@Getter
@NoArgsConstructor
public static class Theme {
@JsonProperty("darkTheme")
private boolean darkTheme;
Theme(boolean darkTheme) {
this.darkTheme = darkTheme;
}
}
}
@@ -0,0 +1,19 @@
package me.vripper.services;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import org.springframework.stereotype.Service;
@Service
public class ThreadPoolService {
@Getter
private final ExecutorService generalExecutor =
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
public void destroy() throws Exception {
generalExecutor.shutdown();
generalExecutor.awaitTermination(5, TimeUnit.SECONDS);
}
}
@@ -0,0 +1,158 @@
package me.vripper.services;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.exception.VripperException;
import me.vripper.jpa.domain.Post;
import me.vripper.tasks.LeaveThanksRunnable;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@Slf4j
public class VGAuthService {
private final ConnectionService cm;
private final SettingsService settingsService;
private final ThreadPoolService threadPoolService;
private final Disposable disposable;
@Getter private final HttpClientContext context = HttpClientContext.create();
@Getter private boolean authenticated = false;
@Getter private String loggedUser = "";
private final EventBus eventBus;
@Autowired
public VGAuthService(
ConnectionService cm,
SettingsService settingsService,
ThreadPoolService threadPoolService,
EventBus eventBus) {
this.cm = cm;
this.settingsService = settingsService;
this.threadPoolService = threadPoolService;
this.eventBus = eventBus;
disposable =
eventBus
.flux()
.filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> authenticate());
}
@PostConstruct
private void init() {
context.setCookieStore(new BasicCookieStore());
authenticate();
}
@PreDestroy
private void destroy() {
disposable.dispose();
}
public void authenticate() {
log.info("Authenticating using ViperGirls credentials");
authenticated = false;
if (!settingsService.getSettings().getVLogin()) {
log.debug("Authentication option is disabled");
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
return;
}
String username = settingsService.getSettings().getVUsername();
String password = settingsService.getSettings().getVPassword();
if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
return;
}
HttpPost postAuth =
cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/login.php?do=login", null);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("vb_login_username", username));
params.add(new BasicNameValuePair("cookieuser", "1"));
params.add(new BasicNameValuePair("do", "login"));
params.add(new BasicNameValuePair("vb_login_md5password", password));
try {
postAuth.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
return;
}
postAuth.addHeader("Referer", settingsService.getSettings().getVProxy());
postAuth.addHeader(
"Host",
settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", ""));
CloseableHttpClient client = cm.getClient().build();
try (CloseableHttpResponse response = client.execute(postAuth, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new VripperException(
String.format(
"Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
}
String responseBody = EntityUtils.toString(response.getEntity());
log.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
EntityUtils.consumeQuietly(response.getEntity());
if (context.getCookieStore().getCookies().stream()
.map(Cookie::getName)
.noneMatch(e -> e.equals("vg_userid"))) {
log.error(
String.format(
"Failed to authenticate user with %s, missing vg_userid cookie",
settingsService.getSettings().getVProxy()));
return;
}
} catch (Exception e) {
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
return;
}
authenticated = true;
loggedUser = username;
log.info(String.format("Authenticated: %s", username));
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
}
public void leaveThanks(Post post) {
threadPoolService
.getGeneralExecutor()
.submit(new LeaveThanksRunnable(post, authenticated, context));
}
}
@@ -0,0 +1,31 @@
package me.vripper.services;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import me.vripper.exception.XpathException;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@Service
public class XpathService {
private final XPath xpath = XPathFactory.newInstance().newXPath();
public Node getAsNode(Node source, String xpathExpression) throws XpathException {
try {
return (Node) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE);
} catch (Exception e) {
throw new XpathException(e);
}
}
public NodeList getAsNodeList(Node source, String xpathExpression) throws XpathException {
try {
return (NodeList) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET);
} catch (Exception e) {
throw new XpathException(e);
}
}
}
@@ -0,0 +1,39 @@
package me.vripper.services.domain;
import java.util.Objects;
import lombok.Getter;
@Getter
public class DownloadSpeed {
private final String speed;
public DownloadSpeed(long bytes) {
speed = formatSI(bytes);
}
private String formatSI(long bytes) {
return humanReadableByteCount(bytes, false);
}
private String humanReadableByteCount(long bytes, boolean si) {
int unit = si ? 1000 : 1024;
if (bytes < unit) return bytes + " B";
int exp = (int) (Math.log(bytes) / Math.log(unit));
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadSpeed that = (DownloadSpeed) o;
return Objects.equals(speed, that.speed);
}
@Override
public int hashCode() {
return Objects.hash(speed);
}
}
@@ -0,0 +1,31 @@
package me.vripper.services.domain;
import java.util.Objects;
import lombok.Getter;
@Getter
public class GlobalState {
private final long running;
private final long remaining;
private final long error;
public GlobalState(long running, long remaining, long error) {
this.running = running;
this.remaining = remaining;
this.error = error;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GlobalState that = (GlobalState) o;
return running == that.running && remaining == that.remaining && error == that.error;
}
@Override
public int hashCode() {
return Objects.hash(running, remaining, error);
}
}
@@ -0,0 +1,35 @@
package me.vripper.services.domain;
import java.util.List;
import lombok.Getter;
@Getter
public class MultiPostItem {
private final String threadId;
private final String postId;
private final int number;
private final String title;
private final int imageCount;
private final String url;
private final List<String> previews;
private final String hosts;
public MultiPostItem(
String threadId,
String postId,
int number,
String title,
int imageCount,
String url,
List<String> previews,
String hosts) {
this.threadId = threadId;
this.postId = postId;
this.number = number;
this.title = title;
this.imageCount = imageCount;
this.previews = previews;
this.url = url;
this.hosts = hosts;
}
}
@@ -0,0 +1,113 @@
package me.vripper.services.domain;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import me.vripper.SpringContext;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.SettingsService;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
public class MultiPostScanHandler extends DefaultHandler {
private final Queued queued;
private final Collection<Host> supportedHosts;
private final SettingsService settingsService;
private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
private final List<MultiPostItem> posts = new ArrayList<>();
private String error;
private List<String> previews = new ArrayList<>();
private String threadTitle;
private String postId;
private String postTitle;
private int imageCount;
private int postCounter;
private int previewCounter = 0;
public MultiPostScanHandler(Queued queued) {
this.queued = queued;
supportedHosts = SpringContext.getBeansOfType(Host.class).values();
settingsService = SpringContext.getBean(SettingsService.class);
}
public MultiPostScanResult getScanResult() {
return new MultiPostScanResult(posts, error);
}
@Override
public void startDocument() {}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "error":
error = attributes.getValue("details");
break;
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
postId = attributes.getValue("id").trim();
postCounter = Integer.parseInt(attributes.getValue("number").trim());
postTitle =
Optional.ofNullable(attributes.getValue("title"))
.map(e -> e.trim().isEmpty() ? null : e.trim())
.orElse(threadTitle);
break;
case "image":
Optional.ofNullable(attributes.getValue("main_url"))
.map(String::trim)
.flatMap(
mainUrl ->
supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
.ifPresent(
host ->
Optional.ofNullable(hostMap.get(host))
.ifPresentOrElse(
AtomicInteger::incrementAndGet,
() -> hostMap.put(host, new AtomicInteger(1))));
if (previewCounter++ < 4) {
Optional.ofNullable(attributes.getValue("thumb_url"))
.map(String::trim)
.ifPresent(previews::add);
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equalsIgnoreCase(qName)) {
if (imageCount != 0) {
posts.add(
new MultiPostItem(
queued.getThreadId(),
postId,
postCounter,
postTitle,
imageCount,
String.format(
"%s/threads/?p=%s&viewfull=1#post%s",
settingsService.getSettings().getVProxy(), postId, postId),
previews,
hostMap.entrySet().stream()
.filter(v -> v.getValue().get() > 0)
.map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")")
.collect(Collectors.joining(", "))));
queued.increment();
}
previewCounter = 0;
previews = new ArrayList<>();
hostMap.clear();
}
}
@Override
public void endDocument() {
queued.done();
}
}
@@ -0,0 +1,90 @@
package me.vripper.services.domain;
import java.io.BufferedInputStream;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.SAXParserFactory;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.ConnectionService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
@Slf4j
public class MultiPostScanParser {
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final Queued queued;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
private final SettingsService settingsService;
public MultiPostScanParser(Queued queued) {
this.queued = queued;
cm = SpringContext.getBean(ConnectionService.class);
VGAuthService = SpringContext.getBean(VGAuthService.class);
settingsService = SpringContext.getBean(SettingsService.class);
}
public MultiPostScanResult parse() throws PostParseException {
log.debug(String.format("Parsing thread %s", queued));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php");
uriBuilder.setParameter("t", queued.getThreadId());
httpGet = cm.buildHttpGet(uriBuilder.build(), null);
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
MultiPostScanHandler multiPostScanHandler = new MultiPostScanHandler(queued);
AtomicReference<Throwable> thr = new AtomicReference<>();
log.debug(String.format("Requesting %s", httpGet));
MultiPostScanResult multiPostScanResult =
Failsafe.with(cm.getRetryPolicy())
.onFailure(e -> thr.set(e.getFailure()))
.get(
() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response =
(CloseableHttpResponse)
connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(
String.format(
"Unexpected response code '%d' for %s",
response.getStatusLine().getStatusCode(), httpGet));
}
try {
factory
.newSAXParser()
.parse(
new BufferedInputStream(response.getEntity().getContent()),
multiPostScanHandler);
return multiPostScanHandler.getScanResult();
} catch (Exception e) {
throw new PostParseException(
String.format("Failed to parse thread %s", queued), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
if (thr.get() != null) {
log.error(String.format("parsing failed for thread %s", queued), thr.get());
throw new PostParseException(thr.get());
}
return multiPostScanResult;
}
}
@@ -0,0 +1,15 @@
package me.vripper.services.domain;
import java.util.List;
import lombok.Getter;
@Getter
public class MultiPostScanResult {
private final List<MultiPostItem> posts;
private final String error;
public MultiPostScanResult(List<MultiPostItem> posts, String error) {
this.posts = posts;
this.error = error;
}
}
@@ -0,0 +1,114 @@
package me.vripper.services.domain;
import java.util.Collection;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.services.SettingsService;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
@Slf4j
class PostScanHandler extends DefaultHandler {
private final Collection<Host> supportedHosts;
private final String threadId;
private final String postId;
private final String postUrl;
private final Set<Image> images = new HashSet<>();
private Set<String> previews = new HashSet<>();
private String threadTitle;
private String forum;
private String userHash;
private int index = 0;
private Post parsedPost;
PostScanHandler(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
supportedHosts = SpringContext.getBeansOfType(Host.class).values();
SettingsService settingsService = SpringContext.getBean(SettingsService.class);
postUrl =
String.format(
"%s/threads/%s/?p=%s&viewfull=1#post%s",
settingsService.getSettings().getVProxy(), this.threadId, this.postId, this.postId);
}
public PostScanResult getParsedPost() {
return new PostScanResult(parsedPost, images);
}
@Override
public void startDocument() {}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "forum":
forum = attributes.getValue("title").trim();
break;
case "user":
userHash = attributes.getValue("hash").trim();
break;
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
String postTitle =
Optional.ofNullable(attributes.getValue("title"))
.map(e -> e.trim().isEmpty() ? null : e.trim())
.orElse(threadTitle);
parsedPost = new Post(postTitle, postUrl, postId, threadId, threadTitle, forum, userHash);
break;
case "image":
index++;
String thumbUrl = attributes.getValue("thumb_url").trim();
if (previews.size() < 4) {
previews.add(thumbUrl);
}
String mainUrl =
Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
if (mainUrl != null) {
Host foundHost =
supportedHosts.stream()
.filter(host -> host.isSupported(mainUrl))
.findFirst()
.orElse(null);
if (foundHost != null) {
log.debug(
String.format(
"Found supported host %s for %s",
foundHost.getClass().getSimpleName(), mainUrl));
images.add(new Image(postId, mainUrl, thumbUrl, foundHost, index));
} else {
log.warn(String.format("unsupported host for %s, skipping", mainUrl));
}
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equalsIgnoreCase(qName)) {
parsedPost.setTotal(images.size());
if (!previews.isEmpty()) {
parsedPost.setPreviews(previews);
}
parsedPost.setHosts(
images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet()));
index = 0;
previews = new HashSet<>();
}
}
}
@@ -0,0 +1,85 @@
package me.vripper.services.domain;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.SAXParserFactory;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.services.ConnectionService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
@Slf4j
public class PostScanParser {
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final String threadId;
private final String postId;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
private final SettingsService settingsService;
public PostScanParser(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
cm = SpringContext.getBean(ConnectionService.class);
VGAuthService = SpringContext.getBean(VGAuthService.class);
settingsService = SpringContext.getBean(SettingsService.class);
}
public PostScanResult parse() throws PostParseException {
log.debug(String.format("Parsing post %s", postId));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php");
uriBuilder.setParameter("p", postId);
httpGet = cm.buildHttpGet(uriBuilder.build(), null);
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
AtomicReference<Throwable> thr = new AtomicReference<>();
PostScanHandler postScanHandler = new PostScanHandler(threadId, postId);
log.debug(String.format("Requesting %s", httpGet));
PostScanResult post = getPost(httpGet, postScanHandler, thr);
if (thr.get() != null) {
log.error(
String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
throw new PostParseException(thr.get());
}
return post;
}
private PostScanResult getPost(
HttpGet httpGet, PostScanHandler postScanHandler, AtomicReference<Throwable> thr) {
return Failsafe.with(cm.getRetryPolicy())
.onFailure(e -> thr.set(e.getFailure()))
.get(
() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response =
(CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(
String.format(
"Unexpected response code '%d' for %s",
response.getStatusLine().getStatusCode(), httpGet));
}
factory.newSAXParser().parse(response.getEntity().getContent(), postScanHandler);
EntityUtils.consumeQuietly(response.getEntity());
return postScanHandler.getParsedPost();
}
});
}
}
@@ -0,0 +1,23 @@
package me.vripper.services.domain;
import java.util.Optional;
import java.util.Set;
import lombok.Getter;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
public class PostScanResult {
private final Post post;
@Getter private final Set<Image> images;
public PostScanResult(Post post, Set<Image> images) {
this.post = post;
this.images = images;
}
public Optional<Post> getPost() {
return Optional.ofNullable(post);
}
}
@@ -0,0 +1,108 @@
package me.vripper.services.domain;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Getter
@Setter
@Slf4j
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Settings implements Cloneable {
@JsonProperty("downloadPath")
private String downloadPath;
@JsonProperty("maxThreads")
private Integer maxThreads;
@JsonProperty("maxTotalThreads")
private Integer maxTotalThreads;
@JsonProperty("autoStart")
private Boolean autoStart;
@JsonProperty("vLogin")
private Boolean vLogin;
@JsonProperty("vUsername")
private String vUsername;
@JsonProperty("vPassword")
private String vPassword;
@JsonProperty("vThanks")
private Boolean vThanks;
@JsonProperty("desktopClipboard")
private Boolean desktopClipboard;
@JsonProperty("forceOrder")
private Boolean forceOrder;
@JsonProperty("subLocation")
private Boolean subLocation;
@JsonProperty("threadSubLocation")
private Boolean threadSubLocation;
@JsonProperty("clearCompleted")
private Boolean clearCompleted;
@JsonProperty("darkTheme")
private Boolean darkTheme;
@JsonProperty("appendPostId")
private Boolean appendPostId;
@JsonProperty("leaveThanksOnStart")
private Boolean leaveThanksOnStart;
@JsonProperty("connectionTimeout")
private Integer connectionTimeout;
@JsonProperty("maxAttempts")
private Integer maxAttempts;
@JsonProperty("vProxy")
private String vProxy;
@JsonProperty("maxEventLog")
private Integer maxEventLog;
@Override
public Object clone() {
Settings clone;
try {
clone = (Settings) super.clone();
} catch (CloneNotSupportedException e) {
log.error(e.getMessage(), e);
clone = new Settings();
}
clone.downloadPath = downloadPath;
clone.maxThreads = maxThreads;
clone.maxTotalThreads = maxTotalThreads;
clone.autoStart = autoStart;
clone.vLogin = vLogin;
clone.vUsername = vUsername;
clone.vPassword = vPassword;
clone.vThanks = vThanks;
clone.desktopClipboard = desktopClipboard;
clone.forceOrder = forceOrder;
clone.subLocation = subLocation;
clone.threadSubLocation = threadSubLocation;
clone.clearCompleted = clearCompleted;
clone.darkTheme = darkTheme;
clone.appendPostId = appendPostId;
clone.leaveThanksOnStart = leaveThanksOnStart;
clone.connectionTimeout = connectionTimeout;
clone.maxAttempts = maxAttempts;
clone.vProxy = vProxy;
clone.maxEventLog = maxEventLog;
return clone;
}
}
@@ -0,0 +1,137 @@
package me.vripper.tasks;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import static me.vripper.jpa.domain.LogEvent.Status.PROCESSING;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.download.DownloadService;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.DataService;
import me.vripper.services.MetadataService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import me.vripper.services.domain.PostScanParser;
import me.vripper.services.domain.PostScanResult;
@Slf4j
public class AddPostRunnable implements Runnable {
private final String postId;
private final String threadId;
private final DataService dataService;
private final MetadataService metadataService;
private final SettingsService settingsService;
private final VGAuthService VGAuthService;
private final LogEvent logEvent;
private final ILogEventRepository eventRepository;
private final String link;
private final DownloadService downloadService;
public AddPostRunnable(String postId, String threadId) {
this.postId = postId;
this.threadId = threadId;
this.dataService = SpringContext.getBean(DataService.class);
this.metadataService = SpringContext.getBean(MetadataService.class);
this.settingsService = SpringContext.getBean(SettingsService.class);
this.downloadService = SpringContext.getBean(DownloadService.class);
this.VGAuthService = SpringContext.getBean(VGAuthService.class);
this.eventRepository = SpringContext.getBean(ILogEventRepository.class);
link =
settingsService.getSettings().getVProxy()
+ String.format("/threads/%s?%s", threadId, (postId != null ? "p=" + postId : ""));
logEvent =
new LogEvent(
LogEvent.Type.POST,
LogEvent.Status.PENDING,
LocalDateTime.now(),
String.format("Processing %s", link));
eventRepository.save(logEvent);
}
@Override
public void run() {
try {
logEvent.setStatus(PROCESSING);
eventRepository.update(logEvent);
if (dataService.exists(postId)) {
log.warn(String.format("skipping %s, already loaded", postId));
logEvent.setMessage(String.format("Gallery %s is already loaded", link));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
return;
}
PostScanParser postScanParser = new PostScanParser(threadId, postId);
PostScanResult postScanResult;
try {
postScanResult = postScanParser.parse();
} catch (PostParseException e) {
String error = String.format("parsing failed for gallery %s", link);
log.error(error, e);
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
return;
}
if (postScanResult.getPost().isEmpty()) {
String error = String.format("Gallery %s contains no galleries", link);
log.error(error);
logEvent.setMessage(error);
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
return;
}
if (postScanResult.getImages().isEmpty()) {
String error = String.format("Gallery %s contains no images to download", link);
log.error(error);
logEvent.setMessage(error);
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
return;
}
Post post = postScanResult.getPost().get();
Set<Image> images = postScanResult.getImages();
dataService.newPost(post, images);
metadataService.startFetchingMetadata(post);
if (settingsService.getSettings().getAutoStart()) {
log.debug("Auto start downloads option is enabled");
post.setStatus(Status.PENDING);
downloadService.enqueue(Map.of(post, images));
log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
} else {
post.setStatus(Status.STOPPED);
log.debug("Auto start downloads option is disabled");
}
if (settingsService.getSettings().getLeaveThanksOnStart() != null
&& !settingsService.getSettings().getLeaveThanksOnStart()) {
VGAuthService.leaveThanks(post);
}
dataService.updatePostStatus(post.getStatus(), post.getId());
logEvent.setMessage(
String.format("Gallery %s is successfully added to download queue", link));
logEvent.setStatus(LogEvent.Status.DONE);
eventRepository.update(logEvent);
} catch (Exception e) {
String error = String.format("Error when adding gallery %s", link);
log.error(error, e);
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
}
}
}
@@ -0,0 +1,102 @@
package me.vripper.tasks;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import static me.vripper.jpa.domain.LogEvent.Status.PROCESSING;
import java.time.LocalDateTime;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.DataService;
import me.vripper.services.PostService;
import me.vripper.services.ThreadPoolService;
import me.vripper.services.domain.MultiPostScanResult;
@Slf4j
public class AddQueuedRunnable implements Runnable {
private final Queued queued;
private final ThreadPoolService threadPoolService;
private final DataService dataService;
private final ILogEventRepository eventRepository;
private final PostService postService;
private final LogEvent logEvent;
public AddQueuedRunnable(Queued queued) {
this.queued = queued;
threadPoolService = SpringContext.getBean(ThreadPoolService.class);
dataService = SpringContext.getBean(DataService.class);
eventRepository = SpringContext.getBean(ILogEventRepository.class);
postService = SpringContext.getBean(PostService.class);
logEvent =
new LogEvent(
LogEvent.Type.QUEUED,
LogEvent.Status.PENDING,
LocalDateTime.now(),
String.format("Processing multi-post link %s", queued.getLink()));
eventRepository.save(logEvent);
}
@Override
public void run() {
try {
logEvent.setStatus(PROCESSING);
eventRepository.update(logEvent);
MultiPostScanResult multiPostScanResult = postService.get(queued);
if (multiPostScanResult == null) {
String message = String.format("Fetching multi-post link %s failed", queued.getLink());
logEvent.setStatus(ERROR);
logEvent.setMessage(message);
eventRepository.update(logEvent);
return;
} else if (multiPostScanResult.getError() != null) {
String message =
"Nothing found for " + queued.getLink() + "\n" + multiPostScanResult.getError();
logEvent.setStatus(ERROR);
logEvent.setMessage(message);
eventRepository.update(logEvent);
return;
} else if (multiPostScanResult.getPosts().isEmpty()) {
String message = "Nothing found for " + queued.getLink();
logEvent.setStatus(ERROR);
logEvent.setMessage(message);
eventRepository.update(logEvent);
return;
}
queued.setTotal(multiPostScanResult.getPosts().size());
queued.done();
if (multiPostScanResult.getPosts().size() == 1) {
threadPoolService
.getGeneralExecutor()
.submit(
new AddPostRunnable(
multiPostScanResult.getPosts().get(0).getPostId(),
multiPostScanResult.getPosts().get(0).getThreadId()));
logEvent.setStatus(LogEvent.Status.DONE);
logEvent.setMessage(String.format("Link %s is added to download queue", queued.getLink()));
} else {
if (dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) {
dataService.newQueueLink(queued);
logEvent.setStatus(LogEvent.Status.DONE);
logEvent.setMessage(
String.format("Link %s is added to multi-post links", queued.getLink()));
} else {
log.info(String.format("Link %s is already loaded", queued.getLink()));
logEvent.setStatus(LogEvent.Status.ERROR);
logEvent.setMessage(
String.format("%s has already been added to multi-post links", queued.getLink()));
}
}
eventRepository.update(logEvent);
} catch (Exception e) {
String error = String.format("Error when adding multi-post link %s", queued.getLink());
log.error(error, e);
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
}
}
}
@@ -0,0 +1,133 @@
package me.vripper.tasks;
import java.io.UnsupportedEncodingException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.ConnectionService;
import me.vripper.services.DataService;
import me.vripper.services.SettingsService;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
@Slf4j
public class LeaveThanksRunnable implements Runnable {
private final ConnectionService cm;
private final DataService dataService;
private final HttpClientContext context;
private final SettingsService settingsService;
private final ILogEventRepository eventRepository;
private final boolean authenticated;
private final Post post;
private final LogEvent logEvent;
public LeaveThanksRunnable(Post post, boolean authenticated, HttpClientContext context) {
this.post = post;
this.authenticated = authenticated;
this.context = context;
cm = SpringContext.getBean(ConnectionService.class);
dataService = SpringContext.getBean(DataService.class);
eventRepository = SpringContext.getBean(ILogEventRepository.class);
settingsService = SpringContext.getBean(SettingsService.class);
logEvent =
new LogEvent(
LogEvent.Type.THANKS,
LogEvent.Status.PENDING,
LocalDateTime.now(),
String.format("Leaving thanks for %s", post.getUrl()));
eventRepository.save(logEvent);
}
@Override
public void run() {
try {
logEvent.setStatus(LogEvent.Status.PROCESSING);
eventRepository.update(logEvent);
if (!settingsService.getSettings().getVLogin()) {
logEvent.setMessage(
String.format(
"Will not send a like for %s\nAuthentication with ViperGirls option is disabled",
post.getUrl()));
logEvent.setStatus(LogEvent.Status.DONE);
eventRepository.update(logEvent);
return;
}
if (!settingsService.getSettings().getVThanks()) {
logEvent.setMessage(
String.format(
"Will not send a like for %s\nLeave thanks option is disabled", post.getUrl()));
logEvent.setStatus(LogEvent.Status.DONE);
eventRepository.update(logEvent);
return;
}
if (!authenticated) {
logEvent.setMessage(
String.format("Will not send a like for %s\nYou are not authenticated", post.getUrl()));
logEvent.setStatus(LogEvent.Status.ERROR);
eventRepository.update(logEvent);
return;
}
if (post.isThanked()) {
logEvent.setMessage(
String.format("Will not send a like for %s\nAlready left a thanks", post.getUrl()));
logEvent.setStatus(LogEvent.Status.DONE);
eventRepository.update(logEvent);
return;
}
HttpPost postThanks =
cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/post_thanks.php", null);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("do", "post_thanks_add"));
params.add(new BasicNameValuePair("using_ajax", "1"));
params.add(new BasicNameValuePair("p", post.getPostId()));
params.add(new BasicNameValuePair("securitytoken", post.getSecurityToken()));
try {
postThanks.setEntity(new UrlEncodedFormEntity(params));
} catch (UnsupportedEncodingException e) {
String error = String.format("Request error for %s", post.getUrl());
log.error(error, e);
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
logEvent.setStatus(LogEvent.Status.ERROR);
eventRepository.update(logEvent);
return;
}
postThanks.addHeader("Referer", settingsService.getSettings().getVProxy());
postThanks.addHeader(
"Host",
settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", ""));
CloseableHttpClient client = cm.getClient().build();
try (CloseableHttpResponse response = client.execute(postThanks, context)) {
if (response.getStatusLine().getStatusCode() / 100 == 2) {
post.setThanked(true);
dataService.updatePostThanks(post.isThanked(), post.getId());
}
EntityUtils.consumeQuietly(response.getEntity());
}
logEvent.setStatus(LogEvent.Status.DONE);
eventRepository.update(logEvent);
} catch (Exception e) {
String error = String.format("Failed to leave a thanks for %s", post.getUrl());
log.error(error, e);
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
logEvent.setStatus(LogEvent.Status.ERROR);
eventRepository.update(logEvent);
}
}
}
@@ -0,0 +1,108 @@
package me.vripper.tasks;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.impl.LogEventRepository;
import me.vripper.services.PostService;
import me.vripper.services.SettingsService;
import org.springframework.lang.NonNull;
@Slf4j
public class LinkScanRunnable implements Runnable {
private static final Object LOCK = new Object();
private final List<String> urlList;
private final SettingsService settingsService;
private final PostService postService;
private final LogEventRepository eventRepository;
private final LogEvent logEvent;
public LinkScanRunnable(@NonNull List<String> urlList) {
this.urlList = urlList;
settingsService = SpringContext.getBean(SettingsService.class);
postService = SpringContext.getBean(PostService.class);
eventRepository = SpringContext.getBean(LogEventRepository.class);
logEvent =
new LogEvent(
LogEvent.Type.SCAN,
LogEvent.Status.PENDING,
LocalDateTime.now(),
"Links to scan:\n\t" + String.join("\n\t", urlList));
eventRepository.save(logEvent);
}
@Override
public void run() {
synchronized (LOCK) {
try {
logEvent.setStatus(LogEvent.Status.PROCESSING);
eventRepository.update(logEvent);
ArrayList<Queued> queuedList = new ArrayList<>();
List<String> unsupported = new ArrayList<>();
List<String> unrecognized = new ArrayList<>();
for (String url : urlList) {
log.debug(String.format("Starting to process thread: %s", url));
if (!url.startsWith(settingsService.getSettings().getVProxy())) {
log.error(String.format("Unsupported link %s", url));
unsupported.add(url);
continue;
}
String threadId, postId;
Matcher m =
Pattern.compile(
Pattern.quote(settingsService.getSettings().getVProxy())
+ "/threads/(\\d+)((.*p=)(\\d+))?")
.matcher(url);
if (m.find()) {
threadId = m.group(1);
postId = m.group(4);
} else {
log.error(String.format("Cannot retrieve thread id from URL %s", url));
unrecognized.add(url);
continue;
}
queuedList.add(new Queued(url, threadId, postId));
}
StringBuilder errorMessage = new StringBuilder();
if (!unsupported.isEmpty()) {
errorMessage
.append("Unsupported links:\n\t")
.append(String.join("\n\t", unsupported))
.append("\n\n");
}
if (!unrecognized.isEmpty()) {
errorMessage
.append("Unrecognized links:\n\t")
.append(String.join("\n\t", unrecognized))
.append("\n\n");
}
postService.processMultiPost(queuedList);
if (!unsupported.isEmpty() || !unrecognized.isEmpty()) {
logEvent.setStatus(LogEvent.Status.ERROR);
logEvent.setMessage("Some links failed to be scanned: \n" + errorMessage);
} else {
logEvent.setStatus(LogEvent.Status.DONE);
}
eventRepository.update(logEvent);
} catch (Exception e) {
String error = "Error when scanning links";
log.error(error, e);
logEvent.setMessage(error + "\n" + Utils.throwableToString(e));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
}
}
}
}
@@ -0,0 +1,245 @@
package me.vripper.tasks;
import static me.vripper.jpa.domain.LogEvent.Status.ERROR;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.download.DownloadJob;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.*;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Slf4j
public class MetadataRunnable implements Runnable {
private static final List<String> dictionary =
Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
@Getter private final Post post;
private final DataService dataService;
private final ILogEventRepository eventRepository;
private final LogEvent logEvent;
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
private final XpathService xpathService;
private final HttpClientContext context = HttpClientContext.create();
private final MetadataService metadataService;
private volatile boolean stopped = false;
@Getter private volatile boolean finished = false;
public MetadataRunnable(@NonNull Post post) {
this.post = post;
dataService = SpringContext.getBean(DataService.class);
eventRepository = SpringContext.getBean(ILogEventRepository.class);
cm = SpringContext.getBean(ConnectionService.class);
VGAuthService vgAuthService = SpringContext.getBean(VGAuthService.class);
htmlProcessorService = SpringContext.getBean(HtmlProcessorService.class);
xpathService = SpringContext.getBean(XpathService.class);
metadataService = SpringContext.getBean(MetadataService.class);
context.setCookieStore(vgAuthService.getContext().getCookieStore());
context.setAttribute(
DownloadJob.ContextAttributes.OPEN_CONNECTION.toString(),
Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
logEvent =
new LogEvent(
LogEvent.Type.METADATA,
LogEvent.Status.PENDING,
LocalDateTime.now(),
"Fetching metadata for " + post.getUrl());
eventRepository.save(logEvent);
}
@Override
public void run() {
try {
logEvent.setStatus(LogEvent.Status.PROCESSING);
eventRepository.update(logEvent);
Metadata metadata = fetchMetadata(post.getPostId(), post.getThreadId(), post.getUrl());
if (metadata != null && !stopped) {
dataService.setMetadata(post, metadata);
logEvent.setStatus(LogEvent.Status.DONE);
} else {
logEvent.setStatus(ERROR);
logEvent.setMessage(String.format("Fetching metadata for %s failed", post.getUrl()));
}
eventRepository.update(logEvent);
} catch (Exception e) {
String message = String.format("Failed to fetch metadata for %s", post.getUrl());
log.error(message, e);
logEvent.setMessage(message + "\n" + Utils.throwableToString(e));
logEvent.setStatus(ERROR);
eventRepository.update(logEvent);
} finally {
if (stopped) {
String message = String.format("Fetching metadata for %s interrupted", post.getUrl());
logEvent.setStatus(LogEvent.Status.DONE);
logEvent.setMessage(message);
eventRepository.update(logEvent);
}
finished = true;
metadataService.stopFetchingMetadata(List.of(post.getPostId()));
}
}
private Metadata fetchMetadata(String postId, String threadId, String url) {
HttpGet httpGet = cm.buildHttpGet(url, context);
AtomicReference<Metadata> metadataReference = new AtomicReference<>();
Failsafe.with(cm.getRetryPolicy())
.onFailure(
e ->
log.error(
String.format("Error occurred when getting post metadata, postId %s", postId),
e.getFailure()))
.run(
() -> {
if (stopped) {
return;
}
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response =
(CloseableHttpResponse) connection.execute(httpGet, context)) {
if (stopped) {
return;
}
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(
String.format(
"Unexpected response code '%d' for %s",
response.getStatusLine().getStatusCode(), httpGet));
}
try {
Document document =
htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
if (stopped) {
return;
}
Node postNode =
xpathService.getAsNode(
document,
String.format(
"//li[@id='post_%s']/div[contains(@class, 'postdetails')]", postId));
if (stopped) {
return;
}
String postedBy =
xpathService
.getAsNode(
postNode,
"./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font")
.getTextContent()
.trim();
if (stopped) {
return;
}
Metadata metadata = new Metadata();
metadata.setPostedBy(postedBy);
if (stopped) {
return;
}
Node node =
xpathService.getAsNode(
document, String.format("//div[@id='post_message_%s']", postId));
metadata.setResolvedNames(findTitleInContent(node));
metadata.setPostId(postId);
if (stopped) {
return;
}
metadataReference.set(metadata);
} catch (Exception e) {
if (stopped) {
log.warn(e.getMessage(), e);
return;
}
throw new PostParseException(
String.format("Failed to parse thread %s, post %s", threadId, postId), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
return metadataReference.get();
}
private List<String> findTitleInContent(Node node) {
List<String> altTitle = new ArrayList<>();
findTitle(node, altTitle, new AtomicBoolean(true));
return altTitle.stream().distinct().collect(Collectors.toList());
}
private void findTitle(Node node, List<String> altTitle, AtomicBoolean keepGoing) {
if (!keepGoing.get()) {
return;
}
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
keepGoing.set(false);
return;
}
if (node.getNodeType() == Node.ELEMENT_NODE) {
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
Node item = node.getChildNodes().item(i);
findTitle(item, altTitle, keepGoing);
if (!keepGoing.get()) {
return;
}
}
} else if (node.getNodeType() == Node.TEXT_NODE) {
String text = node.getTextContent().trim();
if (!text.isBlank()
&& dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
altTitle.add(text);
}
}
}
public void stop() {
this.stopped = true;
List<AbstractExecutionAwareRequest> requests =
(List<AbstractExecutionAwareRequest>)
this.context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
}
}
@@ -0,0 +1,27 @@
package me.vripper.web.restendpoints;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.repositories.ILogEventRepository;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@CrossOrigin(value = "*")
public class EventLogRestEndpoint {
private final ILogEventRepository eventRepository;
public EventLogRestEndpoint(ILogEventRepository eventRepository) {
this.eventRepository = eventRepository;
}
@GetMapping("/events/clear")
@ResponseStatus(value = HttpStatus.OK)
public void clear() {
eventRepository.deleteAll();
}
}
@@ -0,0 +1,267 @@
package me.vripper.web.restendpoints;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.download.DownloadService;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.*;
import me.vripper.services.domain.MultiPostItem;
import me.vripper.services.domain.MultiPostScanResult;
import me.vripper.tasks.AddPostRunnable;
import me.vripper.tasks.LinkScanRunnable;
import me.vripper.web.restendpoints.domain.*;
import me.vripper.web.restendpoints.exceptions.BadRequestException;
import me.vripper.web.restendpoints.exceptions.NotFoundException;
import me.vripper.web.restendpoints.exceptions.ServerErrorException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
@Slf4j
@RestController
@CrossOrigin(value = "*")
public class PostRestEndpoint {
private static final Object LOCK = new Object();
private final DataService dataService;
private final PathService pathService;
private final DownloadService downloadService;
private final PostService postService;
private final ThreadPoolService threadPoolService;
private final SettingsService settingsService;
@Autowired
public PostRestEndpoint(
DataService dataService,
PathService pathService,
DownloadService downloadService,
PostService postService,
ThreadPoolService threadPoolService,
SettingsService settingsService) {
this.dataService = dataService;
this.pathService = pathService;
this.downloadService = downloadService;
this.postService = postService;
this.threadPoolService = threadPoolService;
this.settingsService = settingsService;
}
@PostMapping("/post")
@ResponseStatus(code = HttpStatus.OK)
public void processPost(@RequestBody ThreadUrl _url) {
synchronized (LOCK) {
if (_url.getUrl() == null || _url.getUrl().isBlank()) {
log.error("Cannot process empty requests");
throw new BadRequestException("Cannot process empty requests");
}
List<String> urlList =
Arrays.stream(_url.getUrl().split("\\r?\\n"))
.map(String::trim)
.filter(e -> !e.isEmpty())
.collect(Collectors.toList());
threadPoolService.getGeneralExecutor().submit(new LinkScanRunnable(urlList));
}
}
@PostMapping("/post/restart")
@ResponseStatus(value = HttpStatus.OK)
public void restartPost(@RequestBody @NonNull List<PostId> postIds) {
synchronized (LOCK) {
downloadService.restartAll(
postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
}
}
@PostMapping("/post/add")
@ResponseStatus(value = HttpStatus.OK)
public void addPost(@RequestBody List<PostToAdd> posts) {
synchronized (LOCK) {
for (PostToAdd post : posts) {
threadPoolService
.getGeneralExecutor()
.submit(new AddPostRunnable(post.getPostId(), post.getThreadId()));
}
}
}
@GetMapping("/post/path/{postId}")
@ResponseStatus(value = HttpStatus.OK)
public DownloadPath folderPath(@PathVariable("postId") String postId) {
synchronized (LOCK) {
return getDownloadPath(postId);
}
}
private DownloadPath getDownloadPath(String postId) {
Optional<Post> _post = dataService.findPostByPostId(postId);
if (_post.isPresent()) {
Post post = _post.get();
if (post.getDownloadDirectory() == null) {
log.error("Download has not been started yet for this post");
throw new NotFoundException("Download has not been started yet for this post");
} else {
return new DownloadPath(
pathService.calcDownloadDirectory(post, settingsService.getSettings()).getPath());
}
} else {
log.error(String.format("Unable to find post with postId = %s", postId));
throw new NotFoundException(String.format("Unable to find post with postId = %s", postId));
}
}
@PostMapping("/post/restart/all")
@ResponseStatus(value = HttpStatus.OK)
public void restartPost() {
synchronized (LOCK) {
downloadService.restartAll(null);
}
}
@PostMapping("/post/stop")
@ResponseStatus(value = HttpStatus.OK)
public void stop(@RequestBody @NonNull List<PostId> postIds) {
synchronized (LOCK) {
downloadService.stopAll(postIds.stream().map(PostId::getPostId).collect(Collectors.toList()));
}
}
@PostMapping("/post/stop/all")
@ResponseStatus(value = HttpStatus.OK)
public void stopAll() {
synchronized (LOCK) {
downloadService.stopAll(null);
}
}
@PostMapping("/post/remove")
@ResponseStatus(value = HttpStatus.OK)
public List<RemoveResult> remove(@RequestBody @NonNull List<PostId> postIds) {
synchronized (LOCK) {
List<RemoveResult> result = new ArrayList<>();
List<String> collect =
postIds.stream()
.map(PostId::getPostId)
.peek(e -> result.add(new RemoveResult(e)))
.collect(Collectors.toList());
downloadService.stopAll(collect);
dataService.removeAll(collect);
return result;
}
}
@PostMapping("/post/rename")
@ResponseStatus(value = HttpStatus.OK)
public List<AltPostName> rename(@RequestBody @NonNull List<AltPostName> postToRename) {
synchronized (LOCK) {
renamePosts(postToRename);
return postToRename;
}
}
private void renamePosts(@RequestBody @NonNull List<AltPostName> postToRename) {
synchronized (LOCK) {
for (AltPostName altPostName : postToRename) {
try {
pathService.rename(
altPostName.getPostId(), altPostName.getAltName(), settingsService.getSettings());
} catch (Exception e) {
log.error(
String.format("Failed to rename post with postId = %s", altPostName.getPostId()), e);
throw new ServerErrorException(e.getMessage());
}
}
}
}
@PostMapping("/post/rename/first")
@ResponseStatus(value = HttpStatus.OK)
public List<PostId> renameFirst(@RequestBody @NonNull List<PostId> postToRename) {
synchronized (LOCK) {
renamePostsToFirst(postToRename);
return postToRename;
}
}
public void renamePostsToFirst(@RequestBody @NonNull List<PostId> postToRename) {
synchronized (LOCK) {
for (PostId postId : postToRename) {
Optional<Metadata> _metadata = dataService.findMetadataByPostId(postId.getPostId());
if (_metadata.isPresent()) {
Metadata metadata = _metadata.get();
if (metadata.getResolvedNames() != null) {
List<String> resolvedNames = metadata.getResolvedNames();
if (!resolvedNames.isEmpty()) {
String altTitle = resolvedNames.get(0);
try {
pathService.rename(postId.getPostId(), altTitle, settingsService.getSettings());
} catch (Exception e) {
log.error(
String.format("Failed to rename post with postId = %s", postId.getPostId()), e);
throw new ServerErrorException(e.getMessage());
}
}
}
}
}
}
}
@PostMapping("/post/clear/all")
@ResponseStatus(value = HttpStatus.OK)
public RemoveAllResult clearAll() {
synchronized (LOCK) {
return new RemoveAllResult(dataService.clearCompleted());
}
}
@GetMapping("/grab/{threadId}")
@ResponseStatus(value = HttpStatus.OK)
public List<MultiPostItem> grab(@PathVariable("threadId") @NonNull String threadId) {
synchronized (LOCK) {
try {
Queued queued =
dataService
.findQueuedByThreadId(threadId)
.orElseThrow(
() ->
new NotFoundException(
String.format("Unable to find links for threadId = %s", threadId)));
MultiPostScanResult multiPostScanResult = postService.get(queued);
if (multiPostScanResult == null) {
log.error(String.format("Failed to get links for threadId = %s", threadId));
throw new ServerErrorException(
String.format("Failed to get links for threadId = %s", threadId));
} else {
return multiPostScanResult.getPosts();
}
} catch (Exception e) {
throw new ServerErrorException(
String.format("Failed to get links for threadId = %s, %s", threadId, e.getMessage()));
}
}
}
@PostMapping("/grab/remove")
@ResponseStatus(value = HttpStatus.OK)
public ThreadId grabRemove(@RequestBody @NonNull ThreadId threadId) {
synchronized (LOCK) {
postService.remove(threadId.getThreadId());
return threadId;
}
}
@GetMapping("/grab/clear")
@ResponseStatus(value = HttpStatus.OK)
public void grabClear() {
synchronized (LOCK) {
dataService.clearQueueLinks();
}
}
}
@@ -0,0 +1,65 @@
package me.vripper.web.restendpoints;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.ValidationException;
import me.vripper.services.SettingsService;
import me.vripper.services.domain.Settings;
import me.vripper.web.restendpoints.exceptions.BadRequestException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
@RestController
@Slf4j
@CrossOrigin(value = "*")
public class SettingsRestEndpoint {
private final SettingsService settingsService;
@Autowired
public SettingsRestEndpoint(SettingsService settingsService) {
this.settingsService = settingsService;
}
@PostMapping("/settings/theme")
@ResponseStatus(value = HttpStatus.OK)
public SettingsService.Theme postTheme(@RequestBody SettingsService.Theme theme) {
this.settingsService.setTheme(theme);
return settingsService.getTheme();
}
@GetMapping("/settings/theme")
@ResponseStatus(value = HttpStatus.OK)
public SettingsService.Theme getTheme() {
return settingsService.getTheme();
}
@PostMapping("/settings")
@ResponseStatus(value = HttpStatus.OK)
public Settings postSettings(@RequestBody Settings settings) {
try {
this.settingsService.check(settings);
} catch (ValidationException e) {
log.error("Invalid settings", e);
throw new BadRequestException(e.getMessage());
}
this.settingsService.newSettings(settings);
return getAppSettingsService();
}
@GetMapping("/settings")
@ResponseStatus(value = HttpStatus.OK)
public Settings getAppSettingsService() {
return settingsService.getSettings();
}
@GetMapping("/settings/proxies")
@ResponseStatus(value = HttpStatus.OK)
public List<String> mirrors() {
return settingsService.getProxies();
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web;
package me.vripper.web.restendpoints;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
@@ -7,8 +7,8 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@Configuration
public class WebSecurityConfig implements WebMvcConfigurer {
@Override
public void addViewControllers(ViewControllerRegistry registry) {
registry.addViewController("/home").setViewName("forward:/");
}
@Override
public void addViewControllers(ViewControllerRegistry registry) {
registry.addViewController("/home").setViewName("forward:/");
}
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
package me.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -10,6 +10,6 @@ import lombok.ToString;
@NoArgsConstructor
@ToString
public class AltPostName {
private String postId;
private String altName;
}
private String postId;
private String altName;
}
@@ -1,4 +1,4 @@
package tn.mnlr.vripper.web.restendpoints.domain.posts;
package me.vripper.web.restendpoints.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -10,9 +10,9 @@ import lombok.ToString;
@NoArgsConstructor
@ToString
public class DownloadPath {
private String path;
private String path;
public DownloadPath(String path) {
this.path = path;
}
}
public DownloadPath(String path) {
this.path = path;
}
}

Some files were not shown because too many files have changed in this diff Show More