mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e18832b154 | ||
|
|
6f166b42c4 | ||
|
|
e8ee579a37 | ||
|
|
41ce99995b | ||
|
|
5caeef0e70 | ||
|
|
96ef0cb234 | ||
|
|
641b3e3e7b | ||
|
|
08de4dc5b0 | ||
|
|
57680a6053 | ||
|
|
53fd14935b | ||
|
|
90af03d68f | ||
|
|
265614585b | ||
|
|
4503d540ed | ||
|
|
b08b730dff | ||
|
|
06d636329e | ||
|
|
85313600fc | ||
|
|
7d451397f9 | ||
|
|
1c94b93597 | ||
|
|
8f09596453 |
+232
-3
@@ -1,5 +1,32 @@
|
||||
# Changelog
|
||||
|
||||
## [3.5.0] - 2021-05-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Download queue bug fix
|
||||
- Save window size and position
|
||||
- Resize confirmation box
|
||||
- Enhance metadata interruption
|
||||
- Add Added on and Order columns
|
||||
|
||||
## [3.4.2] - 2021-05-10
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix potential issue with imagebam host
|
||||
- Enhance download queue logic
|
||||
- Code source enhancements
|
||||
- Fix app icons
|
||||
- Fix race condition bug
|
||||
|
||||
## [3.3.8] - 2021-05-05
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix imagebam host
|
||||
- Fix font path
|
||||
|
||||
## [3.3.6] - 2021-03-20
|
||||
|
||||
### Changed
|
||||
@@ -62,213 +89,311 @@
|
||||
- Set total download to max value
|
||||
|
||||
## [3.2.1] - 2021-01-12
|
||||
|
||||
### Added
|
||||
|
||||
- Add settings for connection timeout
|
||||
- Add settings for maximum attempts
|
||||
|
||||
### Changed
|
||||
|
||||
- Enhance posts rename logic
|
||||
- Dependencies update
|
||||
- Switch reactive from rx-java to reactor
|
||||
|
||||
## [3.2.0] - 2020-11-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix build issue
|
||||
|
||||
## [3.1.0] - 2020-11-16
|
||||
|
||||
### Changed
|
||||
|
||||
- UI revamp
|
||||
- Fix bugs
|
||||
|
||||
## [3.0.11] - 2020-10-31
|
||||
|
||||
### Added
|
||||
|
||||
- Support for vipr.im
|
||||
|
||||
## [3.0.10] - 2020-10-04
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bug on settings update
|
||||
|
||||
## [3.0.9] - 2020-09-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Revert to appending post id to download folders
|
||||
- Fix a bug that lead to adding duplicate posts
|
||||
- Remove old unused settings
|
||||
|
||||
|
||||
## [3.0.8] - 2020-08-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fix which breaks concurrent downloads
|
||||
|
||||
## [3.0.7] - 2020-08-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix MacOs build
|
||||
|
||||
## [3.0.6] - 2020-08-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix TurboImageHost
|
||||
|
||||
## [3.0.5] - 2020-08-17
|
||||
|
||||
### Changed
|
||||
|
||||
- Update packages
|
||||
- Host class refactoring
|
||||
|
||||
## [3.0.4] - 2020-08-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Improve rename UI
|
||||
|
||||
## [3.0.3] - 2020-08-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bugs with electron app
|
||||
|
||||
## [3.0.2] - 2020-08-15
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.1] - 2020-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.0] - 2020-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Major rewrites
|
||||
- Introducing a proper database for persistence
|
||||
|
||||
## [2.12.1] - 2020-05-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix alternative name bug
|
||||
|
||||
## [2.12.0] - 2020-05-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Use new user hash from the api
|
||||
|
||||
## [2.11.7] - 2020-05-08
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bug with paths creation
|
||||
- Move Open Link button
|
||||
- Rename rename all title
|
||||
|
||||
## [2.11.6] - 2020-05-04
|
||||
|
||||
### Changed
|
||||
|
||||
- PNG filenames are now correctly named
|
||||
- Fix issue with special characters in post titles
|
||||
|
||||
## [2.11.5] - 2020-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Reduce rows buffer size
|
||||
|
||||
## [2.11.4] - 2020-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Code cleanup
|
||||
- Dependencies update
|
||||
|
||||
## [2.11.3] - 2020-04-30
|
||||
|
||||
### Changed
|
||||
|
||||
- Source code restructure
|
||||
- UI fixes and enhancements
|
||||
|
||||
### Added
|
||||
|
||||
- Add menu entry to rename a gallery to first alternative title
|
||||
- Add menu on toolbar for massive rename to first alternative title
|
||||
|
||||
## [2.11.2] - 2020-04-26
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [2.11.1] - 2020-04-25
|
||||
|
||||
### Changed
|
||||
|
||||
- Code source maintenance
|
||||
|
||||
### Added
|
||||
|
||||
- Added option to rename gallery
|
||||
|
||||
## [2.11.0] - 2020-03-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Upgrade Electron to v8
|
||||
- Upgrade Angular to v9
|
||||
- App settings are now portable in a json file
|
||||
- General code source maintenance
|
||||
|
||||
### Added
|
||||
|
||||
- Added Support for pixroute.com
|
||||
- Added grab post title from thread option
|
||||
|
||||
## [2.10.8] - 2020-01-25
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix app url
|
||||
|
||||
## [2.10.7] - 2020-01-25
|
||||
|
||||
### Changed
|
||||
|
||||
- Drop SockJs and switch to WebSocket
|
||||
- Autoreconnect front to back
|
||||
|
||||
## [2.10.6] - 2020-01-19
|
||||
|
||||
### Changed
|
||||
|
||||
- Limit cache size
|
||||
|
||||
## [2.10.5] - 2020-01-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [2.10.4] - 2020-01-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Zero value for total max download will disable the limit
|
||||
- Add system notification
|
||||
- Bug fixes
|
||||
|
||||
## [2.10.3] - 2020-01-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Some bug fixes
|
||||
- Enhance download queue logic
|
||||
- Clear cache button
|
||||
- Fix the partial status
|
||||
- Sanitize URLs with spaces
|
||||
|
||||
### Added
|
||||
|
||||
- Support for Postimg host
|
||||
- Support for Imagevenue host
|
||||
|
||||
## [2.10.2] - 2019-12-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Upgrade electron to v7
|
||||
- Bug fix
|
||||
|
||||
## [2.10.1] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fix
|
||||
|
||||
## [2.10.0] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix spring dependency bug
|
||||
|
||||
### Added
|
||||
|
||||
- Add support for imgspice
|
||||
|
||||
## [2.9.0] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix build issue
|
||||
|
||||
## [2.9.0] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Decrease timeout for fast fail
|
||||
|
||||
### Added
|
||||
|
||||
- Allow many hosts to run at the same time
|
||||
- Add title to gallery
|
||||
- Add global concurrent downloads
|
||||
- Add counter badge in link collector
|
||||
|
||||
## [2.8.1] - 2019-12-24
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix issue with path creation
|
||||
|
||||
## [2.8.0] - 2019-12-24
|
||||
|
||||
### Added
|
||||
|
||||
- Photo gallery feature
|
||||
|
||||
## [2.7.1] - 2019-12-21
|
||||
|
||||
### Added
|
||||
|
||||
- Add version to title bar
|
||||
|
||||
## [2.7.0] - 2019-12-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Add missing icon
|
||||
|
||||
### Added
|
||||
|
||||
- Add support for pixxxels
|
||||
|
||||
## [2.6.0] - 2019-12-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Disable download button when nothing selected in Link collector grab menu
|
||||
- Better quality app icon
|
||||
- Implement context menu on download tab
|
||||
@@ -277,143 +402,207 @@
|
||||
- Some UI fixes
|
||||
|
||||
## [2.5.1] - 2019-12-09
|
||||
|
||||
### Changed
|
||||
|
||||
- Disable download button when nothing selected in Link collector grab menu
|
||||
|
||||
### Added
|
||||
|
||||
- Add hosts column in Link collector grab menu
|
||||
|
||||
## [2.5.0] - 2019-12-08
|
||||
|
||||
### Changed
|
||||
|
||||
- Change loading screen
|
||||
- UI Fixes
|
||||
- Fix bug with previews
|
||||
|
||||
### Added
|
||||
|
||||
- Add link collector feature
|
||||
- Add support for multiple links at a time
|
||||
|
||||
## [2.4.1] - 2019-11-26
|
||||
|
||||
### Changed
|
||||
|
||||
- Increase attempts for establishing ws connection between front and back end
|
||||
- Change default download folder to Home folder (installer), App folder (portable)
|
||||
|
||||
## [2.4.0] - 2019-11-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Pimpandhost Host support
|
||||
|
||||
## [2.3.2] - 2019-11-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a bug with the scan component
|
||||
|
||||
## [2.3.1] - 2019-11-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix connexion reset issue
|
||||
- Fix Post Status when autostart option is selected
|
||||
- Add stop all button
|
||||
|
||||
## [2.3.0] - 2019-11-21
|
||||
|
||||
### Changed
|
||||
|
||||
- UI enhancements and overhaul
|
||||
- Stability fixes with HTTP connections
|
||||
- MacOS official support
|
||||
- Use local sockjs and material icons font
|
||||
|
||||
## [2.2.5] - 2019-11-09
|
||||
|
||||
### Changed
|
||||
|
||||
- Exclude thumbs download for imagezilla and pixhost
|
||||
- Change retry on fail logic (may help avoid having download errors)
|
||||
|
||||
## [2.2.4] - 2019-10-05
|
||||
|
||||
### Added
|
||||
|
||||
- AppImage target for linux
|
||||
|
||||
## [2.2.3] - 2019-10-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Strip libjvm.so for debian
|
||||
|
||||
## [2.2.2] - 2019-10-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Update product name
|
||||
|
||||
## [2.2.1] - 2019-10-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Update dependency list for deb package
|
||||
|
||||
## [2.2.0] - 2019-10-01
|
||||
|
||||
### Added
|
||||
|
||||
- Embedd JRE within the app
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix 'Start' on pending items
|
||||
- Upgrade to Java 11
|
||||
|
||||
## [2.1.0] - 2019-09-30
|
||||
|
||||
### Added
|
||||
|
||||
- Add a create a subfolder per thread option
|
||||
|
||||
## [2.0.4] - 2019-09-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix some issues with ImxHost
|
||||
|
||||
## [2.0.3] - 2019-09-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Update deb dependencies
|
||||
|
||||
## [2.0.2] - 2019-09-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Add portable mode
|
||||
|
||||
## [2.0.1] - 2019-09-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Change app name
|
||||
|
||||
## [2.0.0] - 2019-09-22
|
||||
|
||||
### Added
|
||||
|
||||
- Add sub folder option
|
||||
- Add clear finished option
|
||||
|
||||
### Changed
|
||||
|
||||
- UI enhancements
|
||||
- Bug fixes
|
||||
- Change data and log paths
|
||||
|
||||
## [1.6.5] - 2019-09-18
|
||||
|
||||
### Changed
|
||||
|
||||
- electron update
|
||||
|
||||
## [1.6.4] - 2019-09-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Change menu item name
|
||||
|
||||
## [1.6.3] - 2019-09-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Better multithread management
|
||||
- Fix partial state UI
|
||||
|
||||
## [1.6.2] - 2019-09-17
|
||||
|
||||
### Changed
|
||||
|
||||
- Rework UI colors
|
||||
- Fix scan input bug
|
||||
|
||||
## [1.6.1] - 2019-09-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Rework UI
|
||||
- Fix clipboard monitoring
|
||||
|
||||
## [1.6.0] - 2019-09-15
|
||||
|
||||
### Changed
|
||||
|
||||
- Rework the UI
|
||||
- Add a dark theme
|
||||
- Fix the force ordering bug
|
||||
- Other bug fixes
|
||||
|
||||
## [1.5.2] - 2019-09-01
|
||||
|
||||
### Added
|
||||
|
||||
- imx.to host fix
|
||||
|
||||
## [1.5.1] - 2019-08-24
|
||||
|
||||
### Added
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [1.5.0] - 2019-08-24
|
||||
|
||||
### Added
|
||||
|
||||
- Display logged in user
|
||||
- Add post details when clicking on main list items (Status, Post URL, Host)
|
||||
- Add open download location on main list items menu
|
||||
@@ -421,86 +610,126 @@
|
||||
- Add link to image from details list
|
||||
|
||||
## [1.4.10] - 2019-08-15
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a rename issue on windows
|
||||
|
||||
## [1.4.9] - 2019-08-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Update Built-By to be generic
|
||||
|
||||
## [1.4.8] - 2019-08-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Add option to force image ordering
|
||||
|
||||
## [1.4.7] - 2019-08-13
|
||||
|
||||
### Changed
|
||||
- Fixed a bug in image filename extensions
|
||||
|
||||
- Fixed a bug in image filename extensions
|
||||
|
||||
## [1.4.6] - 2019-08-06
|
||||
|
||||
### Changed
|
||||
|
||||
- Fixed a bug when creating image file names
|
||||
- Remove unnecessary logs
|
||||
- Fix data.json init
|
||||
- Fix data.json init
|
||||
|
||||
## [1.4.5] - 2019-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Add min width and height for electron app
|
||||
|
||||
### Added
|
||||
|
||||
- Add download speed to status bar
|
||||
|
||||
## [1.4.4] - 2019-08-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix names for ImgboxHost
|
||||
- Enhance UI
|
||||
- Minor fixes
|
||||
|
||||
### Added
|
||||
|
||||
- Add start and stop button
|
||||
|
||||
## [1.4.3] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Update some messages
|
||||
- Fix scrolling issue
|
||||
|
||||
## [1.4.2] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix layout issue in web mode
|
||||
|
||||
## [1.4.1] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Change title's font
|
||||
|
||||
### Added
|
||||
|
||||
- Add borders
|
||||
|
||||
## [1.4.0] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Better error handling on startup
|
||||
- Better support for linux AppImage
|
||||
|
||||
### Added
|
||||
|
||||
- Clear all completed
|
||||
- Add clipboard support
|
||||
- Add option to remove all posts
|
||||
- Add confirmation on remove
|
||||
|
||||
## [1.3.0-rc0] - 2019-07-04
|
||||
|
||||
### Changed
|
||||
|
||||
- Change default setting for autostart false -> true
|
||||
- Change default setting for max concurrent downloads 1 -> 4
|
||||
|
||||
### Added
|
||||
|
||||
- Desktop app built with electron
|
||||
- Add the option to remove a post
|
||||
|
||||
## [1.2.0] - 2019-06-23
|
||||
|
||||
### Added
|
||||
|
||||
- Add a shutdown button to gracefully shutdown the app
|
||||
- Open and redirect the browser to the app address on startup
|
||||
- Add the option to run the app in headless mode (-Djava.awt.headless=true)
|
||||
|
||||
## [1.1.1] - 2019-06-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a stability issue with progress updates when the app is runninh for a long time
|
||||
|
||||
## [1.0.0] - 2019-06-22
|
||||
|
||||
### Added
|
||||
|
||||
- Add real time communication for progress updates via websocket (SockJS)
|
||||
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
|
||||
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.6</version>
|
||||
<version>3.5.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
<version>2.4.5</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
<modules>
|
||||
<module>vripper-server</module>
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 2.1 KiB |
@@ -7,6 +7,7 @@ const {spawn} = require("child_process");
|
||||
const {ipcMain} = require("electron");
|
||||
const {dialog} = require("electron");
|
||||
const axios = require('axios');
|
||||
const windowStateKeeper = require('electron-window-state');
|
||||
|
||||
// non null value when it is an AppImage
|
||||
const appImageDir = process.env.APPDIR;
|
||||
@@ -29,18 +30,26 @@ process.on("uncaughtException", err => {
|
||||
createWindow = () => {
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
app.setAppUserModelId("tn.mnlr.vripper");
|
||||
app.setAppUserModelId('tn.mnlr.vripper');
|
||||
}
|
||||
|
||||
let icon;
|
||||
if (process.platform === "win32") {
|
||||
if (process.platform === 'win32') {
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
} else if (process.platform === 'linux') {
|
||||
icon = __dirname + '/icon.png';
|
||||
}
|
||||
|
||||
const mainWindowState = windowStateKeeper({
|
||||
defaultWidth: 1024,
|
||||
defaultHeight: 800
|
||||
});
|
||||
|
||||
win = new BrowserWindow({
|
||||
width: 1024,
|
||||
height: 800,
|
||||
x: mainWindowState.x,
|
||||
y: mainWindowState.y,
|
||||
width: mainWindowState.width,
|
||||
height: mainWindowState.height,
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
webPreferences: {
|
||||
@@ -50,6 +59,7 @@ createWindow = () => {
|
||||
icon: icon
|
||||
});
|
||||
|
||||
mainWindowState.manage(win);
|
||||
win.removeMenu();
|
||||
win.setMenu(null);
|
||||
win.setMenuBarVisibility(false);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vripper-electron",
|
||||
"version": "3.3.6",
|
||||
"version": "3.5.0",
|
||||
"description": "A ripper for vipergirls.to built using web technolgies",
|
||||
"main": "main.js",
|
||||
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
|
||||
@@ -51,7 +51,6 @@
|
||||
"synopsis": "vipergirls.to ripper",
|
||||
"category": "Utility",
|
||||
"packageCategory": "Utility",
|
||||
"icon": "icons",
|
||||
"target": [
|
||||
"AppImage"
|
||||
]
|
||||
@@ -77,6 +76,7 @@
|
||||
"cheerio": "1.0.0-rc.3",
|
||||
"copy-dir": "1.3.0",
|
||||
"electron-context-menu": "2.1.0",
|
||||
"electron-window-state": "^5.0.3",
|
||||
"get-port": "5.1.1",
|
||||
"rimraf": "3.0.2",
|
||||
"v8-compile-cache": "2.1.1"
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
<?xml version="1.0"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0">
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.6</version>
|
||||
<version>3.5.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-electron</artifactId>
|
||||
<name>vripper-electron</name>
|
||||
@@ -22,9 +23,6 @@
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>electron</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<buildClassifier>electron</buildClassifier>
|
||||
</properties>
|
||||
|
||||
@@ -18,7 +18,7 @@ fs.writeFileSync('./build/vripper-ui/index.html', $.html());
|
||||
|
||||
console.log('Building runtime environment');
|
||||
rimraf.sync('java-runtime');
|
||||
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
|
||||
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.security.jgss,java.sql,jdk.unsupported,jdk.crypto.ec --output java-runtime');
|
||||
if (process.platform === 'linux') {
|
||||
console.log('Stripping libjvm.so');
|
||||
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
|
||||
|
||||
@@ -18,115 +18,5 @@
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="module" module-name="vripper-server" scope="RUNTIME" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-jersey:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-json:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.core:jackson-core:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.module:jackson-module-parameter-names:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: jakarta.annotation:jakarta.annotation-api:1.3.5" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:9.0.41" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish:jakarta.el:3.0.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:9.0.41" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-validation:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.hibernate.validator:hibernate-validator:6.1.6.Final" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.jboss.logging:jboss-logging:3.4.1.Final" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml:classmate:1.5.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-web:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-beans:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-core:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-jcl:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.core:jersey-server:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.core:jersey-common:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:osgi-resource-locator:1.0.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.activation:jakarta.activation:1.2.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.core:jersey-client:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: jakarta.ws.rs:jakarta.ws.rs-api:2.1.6" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.media:jersey-media-jaxb:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2.external:jakarta.inject:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: jakarta.validation:jakarta.validation-api:2.0.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: jakarta.xml.bind:jakarta.xml.bind-api:2.3.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: jakarta.activation:jakarta.activation-api:1.2.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.containers:jersey-container-servlet-core:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.containers:jersey-container-servlet:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.ext:jersey-bean-validation:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.ext:jersey-spring5:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.inject:jersey-hk2:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:hk2-locator:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2.external:aopalliance-repackaged:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.javassist:javassist:3.25.0-GA" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:hk2:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:hk2-utils:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:hk2-api:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:hk2-core:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:hk2-runlevel:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:class-model:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.ow2.asm:asm:7.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.ow2.asm:asm-analysis:7.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.ow2.asm:asm-commons:7.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.ow2.asm:asm-tree:7.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.ow2.asm:asm-util:7.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.hk2:spring-bridge:2.6.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-context:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-aop:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.media:jersey-media-json-jackson:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jersey.ext:jersey-entity-filtering:2.32" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.11.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-web:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: ch.qos.logback:logback-classic:1.2.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.13.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.logging.log4j:log4j-api:2.13.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.slf4j:jul-to-slf4j:1.7.30" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.yaml:snakeyaml:1.27" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-webmvc:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-expression:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-websocket:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-messaging:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-websocket:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-actuator:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-actuator-autoconfigure:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-actuator:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: io.micrometer:micrometer-core:1.6.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.hdrhistogram:HdrHistogram:2.1.12" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.latencyutils:LatencyUtils:2.0.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-data-jdbc:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.zaxxer:HikariCP:3.4.5" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-jdbc:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.data:spring-data-jdbc:2.1.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.data:spring-data-relational:2.1.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.data:spring-data-commons:2.4.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework:spring-tx:5.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.liquibase:liquibase-core:3.10.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: javax.xml.bind:jaxb-api:2.3.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: javax.activation:javax.activation-api:1.2.0" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.slf4j:slf4j-api:1.7.30" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.hsqldb:hsqldb:2.5.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.httpcomponents:httpclient:4.5.13" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.httpcomponents:httpcore:4.4.14" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: commons-codec:commons-codec:1.15" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: net.sourceforge.htmlcleaner:htmlcleaner:2.22" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.jdom:jdom2:2.0.6" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: io.projectreactor:reactor-core:3.4.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.reactivestreams:reactive-streams:1.0.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: net.jodah:failsafe:2.0.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.guava:guava:28.1-jre" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.guava:failureaccess:1.0.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.code.findbugs:jsr305:3.0.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.checkerframework:checker-qual:2.8.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.errorprone:error_prone_annotations:2.3.2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.google.j2objc:j2objc-annotations:1.3" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.codehaus.mojo:animal-sniffer-annotations:1.18" level="project" />
|
||||
<orderEntry type="module" module-name="vripper-ui" scope="RUNTIME" />
|
||||
</component>
|
||||
</module>
|
||||
+286
-278
File diff suppressed because it is too large
Load Diff
+5
-16
@@ -1,11 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>3.3.6</version>
|
||||
<version>3.5.0</version>
|
||||
</parent>
|
||||
<artifactId>vripper-server</artifactId>
|
||||
<name>vripper-server</name>
|
||||
@@ -16,14 +16,6 @@
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jersey</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-websocket</artifactId>
|
||||
@@ -43,7 +35,6 @@
|
||||
<dependency>
|
||||
<groupId>org.hsqldb</groupId>
|
||||
<artifactId>hsqldb</artifactId>
|
||||
<version>2.5.1</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -64,12 +55,11 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.5.6</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.htmlcleaner</groupId>
|
||||
<artifactId>htmlcleaner</artifactId>
|
||||
<version>2.22</version>
|
||||
<version>2.24</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
@@ -81,9 +71,8 @@
|
||||
<version>2.0.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>28.1-jre</version>
|
||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
||||
<artifactId>caffeine</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
|
||||
@@ -5,29 +5,23 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.event.ContextRefreshedEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tn.mnlr.vripper.jpa.repositories.IRepository;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
import java.util.Set;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
|
||||
@Component
|
||||
public class EventListenerBean {
|
||||
|
||||
@Getter
|
||||
private static boolean init = false;
|
||||
@Getter private static boolean init = false;
|
||||
|
||||
private final DataService dataService;
|
||||
private final Set<IRepository> repositorySet;
|
||||
private final IPostRepository postRepository;
|
||||
|
||||
@Autowired
|
||||
public EventListenerBean(DataService dataService, Set<IRepository> repositorySet) {
|
||||
this.dataService = dataService;
|
||||
this.repositorySet = repositorySet;
|
||||
}
|
||||
@Autowired
|
||||
public EventListenerBean(IPostRepository postRepository) {
|
||||
this.postRepository = postRepository;
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void onApplicationEvent(ContextRefreshedEvent event) {
|
||||
dataService.setDownloadingToStopped();
|
||||
init = true;
|
||||
}
|
||||
@EventListener
|
||||
public void onApplicationEvent(ContextRefreshedEvent event) {
|
||||
postRepository.setDownloadingToStopped();
|
||||
init = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,28 +13,27 @@ import java.util.Map;
|
||||
@Slf4j
|
||||
public class SpringContext implements ApplicationContextAware {
|
||||
|
||||
private static ConfigurableApplicationContext context;
|
||||
private static ConfigurableApplicationContext context;
|
||||
|
||||
public static <T> T getBean(Class<T> beanClass) {
|
||||
return context.getBean(beanClass);
|
||||
public static <T> T getBean(Class<T> beanClass) {
|
||||
return context.getBean(beanClass);
|
||||
}
|
||||
|
||||
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
|
||||
return context.getBeansOfType(beanClass);
|
||||
}
|
||||
|
||||
public static void close() {
|
||||
log.info("Application terminating...");
|
||||
if (context != null) {
|
||||
context.close();
|
||||
}
|
||||
}
|
||||
|
||||
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
|
||||
return context.getBeansOfType(beanClass);
|
||||
}
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext context) throws BeansException {
|
||||
|
||||
public static void close() {
|
||||
log.info("Application terminating...");
|
||||
if (context != null) {
|
||||
context.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext context) throws BeansException {
|
||||
|
||||
// store ApplicationContext reference to access required beans later on
|
||||
SpringContext.context = ((ConfigurableApplicationContext) context);
|
||||
}
|
||||
}
|
||||
// store ApplicationContext reference to access required beans later on
|
||||
SpringContext.context = ((ConfigurableApplicationContext) context);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,15 +9,16 @@ import java.io.StringWriter;
|
||||
@Slf4j
|
||||
public class Utils {
|
||||
|
||||
public static String throwableToString(Throwable th) {
|
||||
try (StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter)) {
|
||||
th.printStackTrace(printWriter);
|
||||
printWriter.flush();
|
||||
stringWriter.flush();
|
||||
return stringWriter.toString();
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to convert throwable to string", e);
|
||||
}
|
||||
return "Cannot display error, please check the log file for details";
|
||||
public static String throwableToString(Throwable th) {
|
||||
try (StringWriter stringWriter = new StringWriter();
|
||||
PrintWriter printWriter = new PrintWriter(stringWriter)) {
|
||||
th.printStackTrace(printWriter);
|
||||
printWriter.flush();
|
||||
stringWriter.flush();
|
||||
return stringWriter.toString();
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to convert throwable to string", e);
|
||||
}
|
||||
return "Cannot display error, please check the log file for details";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,13 +8,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
@Slf4j
|
||||
public class VripperApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
|
||||
SpringApplication.run(VripperApplication.class, args);
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to run the application", e);
|
||||
}
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
|
||||
SpringApplication.run(VripperApplication.class, args);
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to run the application", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.*;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
@@ -32,246 +33,263 @@ import java.util.*;
|
||||
@Slf4j
|
||||
public class DownloadJob implements CheckedRunnable {
|
||||
|
||||
private static final Object LOCK = new Object();
|
||||
private static final int READ_BUFFER_SIZE = 8192;
|
||||
private final DataService dataService;
|
||||
private final PathService pathService;
|
||||
private final ConnectionService cm;
|
||||
private final VGAuthService authService;
|
||||
private final DownloadSpeedService downloadSpeedService;
|
||||
private final SettingsService settingsService;
|
||||
private final HttpClientContext context;
|
||||
@Getter
|
||||
private final Image image;
|
||||
@Getter
|
||||
private final Post post;
|
||||
private volatile boolean stopped = false;
|
||||
@Getter
|
||||
private boolean finished = false;
|
||||
private static final Object LOCK = new Object();
|
||||
private static final int READ_BUFFER_SIZE = 8192;
|
||||
private final DataService dataService;
|
||||
private final PathService pathService;
|
||||
private final ConnectionService cm;
|
||||
private final VGAuthService authService;
|
||||
private final DownloadSpeedService downloadSpeedService;
|
||||
private final Settings settings;
|
||||
private final HttpClientContext context;
|
||||
@Getter private final Image image;
|
||||
@Getter private final Post post;
|
||||
private volatile boolean stopped = false;
|
||||
@Getter private volatile boolean finished = false;
|
||||
|
||||
DownloadJob(Post post, Image image) {
|
||||
this.image = image;
|
||||
this.post = post;
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
pathService = SpringContext.getBean(PathService.class);
|
||||
cm = SpringContext.getBean(ConnectionService.class);
|
||||
authService = SpringContext.getBean(VGAuthService.class);
|
||||
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
|
||||
settingsService = SpringContext.getBean(SettingsService.class);
|
||||
context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
context.setAttribute(ContextAttributes.OPEN_CONNECTION.toString(), Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
|
||||
}
|
||||
DownloadJob(Post post, Image image, Settings settings) {
|
||||
this.image = image;
|
||||
this.post = post;
|
||||
this.settings = settings;
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
pathService = SpringContext.getBean(PathService.class);
|
||||
cm = SpringContext.getBean(ConnectionService.class);
|
||||
authService = SpringContext.getBean(VGAuthService.class);
|
||||
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
|
||||
context = HttpClientContext.create();
|
||||
context.setCookieStore(new BasicCookieStore());
|
||||
context.setAttribute(
|
||||
ContextAttributes.OPEN_CONNECTION.toString(),
|
||||
Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
|
||||
}
|
||||
|
||||
public void download(final Post post, final Image image) throws DownloadException {
|
||||
public void download(final Post post, final Image image) throws DownloadException {
|
||||
|
||||
try {
|
||||
try {
|
||||
|
||||
image.setStatus(Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
image.setStatus(Status.DOWNLOADING);
|
||||
image.setCurrent(0);
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
|
||||
synchronized (LOCK) {
|
||||
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
|
||||
post.setStatus(Status.DOWNLOADING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
}
|
||||
|
||||
// The post may be updated and the download directory might be set by another thread
|
||||
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
|
||||
if (updatedPost.getDownloadDirectory() == null) {
|
||||
pathService.createDefaultPostFolder(updatedPost);
|
||||
}
|
||||
if (settingsService.getSettings().getLeaveThanksOnStart() != null && settingsService.getSettings().getLeaveThanksOnStart()) {
|
||||
authService.leaveThanks(updatedPost);
|
||||
}
|
||||
}
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* HOST SPECIFIC
|
||||
*/
|
||||
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
|
||||
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
|
||||
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
|
||||
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
|
||||
/*
|
||||
* END HOST SPECIFIC
|
||||
*/
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
|
||||
log.debug(String.format("Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
|
||||
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
|
||||
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
|
||||
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
|
||||
httpGet.addHeader("Referer", image.getUrl());
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
|
||||
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
image.setTotal(response.getEntity().getContentLength());
|
||||
dataService.updateImageTotal(image.getTotal(), image.getId());
|
||||
|
||||
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
|
||||
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
|
||||
|
||||
byte[] buffer = new byte[READ_BUFFER_SIZE];
|
||||
int read;
|
||||
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
|
||||
fos.write(buffer, 0, read);
|
||||
image.increase(read);
|
||||
downloadSpeedService.increase(read);
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
}
|
||||
fos.flush();
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
checkImageTypeAndRename(dataService.findById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
throw new DownloadException(e);
|
||||
} finally {
|
||||
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
|
||||
image.setStatus(Status.COMPLETE);
|
||||
} else if (stopped) {
|
||||
image.setStatus(Status.STOPPED);
|
||||
} else {
|
||||
image.setStatus(Status.ERROR);
|
||||
}
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
finished = true;
|
||||
synchronized (LOCK) {
|
||||
if (!post.getStatus().equals(Status.DOWNLOADING)
|
||||
&& !post.getStatus().equals(Status.PARTIAL)) {
|
||||
post.setStatus(Status.DOWNLOADING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
}
|
||||
}
|
||||
|
||||
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
|
||||
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
|
||||
if (!it.hasNext()) {
|
||||
throw new HostException("Image file is not recognized!");
|
||||
}
|
||||
ImageReader reader = it.next();
|
||||
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
|
||||
String imageNameLC = imageName.toLowerCase();
|
||||
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
|
||||
imageName += ".jpg";
|
||||
} else {
|
||||
String toReplace = null;
|
||||
if (imageNameLC.endsWith("_jpg")) {
|
||||
toReplace = "_jpg";
|
||||
} else if (imageNameLC.endsWith("_jpeg")) {
|
||||
toReplace = "_jpeg";
|
||||
}
|
||||
if (toReplace != null) {
|
||||
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
|
||||
}
|
||||
}
|
||||
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
|
||||
String imageNameLC = imageName.toLowerCase();
|
||||
if (!imageNameLC.endsWith("_png")) {
|
||||
imageName += ".png";
|
||||
} else {
|
||||
String toReplace = null;
|
||||
if (imageNameLC.endsWith("_png")) {
|
||||
toReplace = "_png";
|
||||
}
|
||||
if (toReplace != null) {
|
||||
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to guess image format", e);
|
||||
// The post may be updated and the download directory might be set by another thread
|
||||
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
|
||||
if (updatedPost.getDownloadDirectory() == null) {
|
||||
pathService.createDefaultPostFolder(updatedPost, settings);
|
||||
}
|
||||
try {
|
||||
pathService.getDirectoryAccess().lock();
|
||||
File downloadDestinationFolder = pathService.calcDownloadDirectory(post);
|
||||
File outImage = new File(downloadDestinationFolder, (settingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
|
||||
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to rename the image", e);
|
||||
} finally {
|
||||
pathService.getDirectoryAccess().unlock();
|
||||
try {
|
||||
Files.delete(outputFile.toPath());
|
||||
} catch (IOException e) {
|
||||
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
|
||||
}
|
||||
if (settings.getLeaveThanksOnStart() != null && settings.getLeaveThanksOnStart()) {
|
||||
authService.leaveThanks(updatedPost);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* HOST SPECIFIC
|
||||
*/
|
||||
log.debug(
|
||||
String.format(
|
||||
"Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
|
||||
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
|
||||
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
|
||||
log.debug(
|
||||
String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
|
||||
/*
|
||||
* END HOST SPECIFIC
|
||||
*/
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
|
||||
log.debug(
|
||||
String.format(
|
||||
"Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
|
||||
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
|
||||
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
|
||||
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
|
||||
httpGet.addHeader("Referer", image.getUrl());
|
||||
try (CloseableHttpResponse response =
|
||||
(CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
throw new DownloadException(
|
||||
String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
if (stopped) {
|
||||
finished = true;
|
||||
return;
|
||||
}
|
||||
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
|
||||
try (InputStream downloadStream = response.getEntity().getContent();
|
||||
FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
image.setTotal(response.getEntity().getContentLength());
|
||||
dataService.updateImageTotal(image.getTotal(), image.getId());
|
||||
|
||||
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
|
||||
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
|
||||
|
||||
byte[] buffer = new byte[READ_BUFFER_SIZE];
|
||||
int read;
|
||||
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
|
||||
fos.write(buffer, 0, read);
|
||||
image.increase(read);
|
||||
downloadSpeedService.increase(read);
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
}
|
||||
fos.flush();
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
log.debug(String.format("Starting downloading %s", image.getUrl()));
|
||||
download(post, image);
|
||||
checkImageTypeAndRename(
|
||||
dataService.findById(post.getId()).orElseThrow(),
|
||||
outputFile,
|
||||
nameAndUrl.getName(),
|
||||
image.getIndex());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
throw new DownloadException(e);
|
||||
} finally {
|
||||
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
|
||||
image.setStatus(Status.COMPLETE);
|
||||
} else if (stopped) {
|
||||
image.setStatus(Status.STOPPED);
|
||||
} else {
|
||||
image.setStatus(Status.ERROR);
|
||||
}
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index)
|
||||
throws HostException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
|
||||
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
|
||||
if (!it.hasNext()) {
|
||||
throw new HostException("Image file is not recognized!");
|
||||
}
|
||||
ImageReader reader = it.next();
|
||||
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
|
||||
String imageNameLC = imageName.toLowerCase();
|
||||
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
|
||||
imageName += ".jpg";
|
||||
} else {
|
||||
String toReplace = null;
|
||||
if (imageNameLC.endsWith("_jpg")) {
|
||||
toReplace = "_jpg";
|
||||
} else if (imageNameLC.endsWith("_jpeg")) {
|
||||
toReplace = "_jpeg";
|
||||
}
|
||||
if (toReplace != null) {
|
||||
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
|
||||
}
|
||||
}
|
||||
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
|
||||
String imageNameLC = imageName.toLowerCase();
|
||||
if (!imageNameLC.endsWith("_png")) {
|
||||
imageName += ".png";
|
||||
} else {
|
||||
String toReplace = null;
|
||||
if (imageNameLC.endsWith("_png")) {
|
||||
toReplace = "_png";
|
||||
}
|
||||
if (toReplace != null) {
|
||||
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to guess image format", e);
|
||||
}
|
||||
try {
|
||||
pathService.getDirectoryAccess().lock();
|
||||
File downloadDestinationFolder = pathService.calcDownloadDirectory(post, settings);
|
||||
File outImage =
|
||||
new File(
|
||||
downloadDestinationFolder,
|
||||
(settings.getForceOrder() ? String.format("%03d_", index) : "") + imageName);
|
||||
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Failed to rename the image", e);
|
||||
} finally {
|
||||
pathService.getDirectoryAccess().unlock();
|
||||
try {
|
||||
Files.delete(outputFile.toPath());
|
||||
} catch (IOException e) {
|
||||
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
if (stopped) {
|
||||
finished = true;
|
||||
return;
|
||||
}
|
||||
log.debug(String.format("Starting downloading %s", image.getUrl()));
|
||||
download(post, image);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
DownloadJob that = (DownloadJob) o;
|
||||
return Objects.equals(image, that.image) && Objects.equals(post, that.post);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(image, post);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
this.stopped = true;
|
||||
List<AbstractExecutionAwareRequest> requests =
|
||||
(List<AbstractExecutionAwareRequest>)
|
||||
this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
|
||||
if (requests != null) {
|
||||
for (AbstractExecutionAwareRequest request : requests) {
|
||||
request.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum ContextAttributes {
|
||||
OPEN_CONNECTION("OPEN_CONNECTION");
|
||||
|
||||
private final String value;
|
||||
|
||||
ContextAttributes(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
DownloadJob that = (DownloadJob) o;
|
||||
return Objects.equals(image, that.image) &&
|
||||
Objects.equals(post, that.post);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(image, post);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
|
||||
if (requests != null) {
|
||||
for (AbstractExecutionAwareRequest request : requests) {
|
||||
request.abort();
|
||||
}
|
||||
}
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
public enum ContextAttributes {
|
||||
OPEN_CONNECTION("OPEN_CONNECTION");
|
||||
|
||||
private final String value;
|
||||
|
||||
ContextAttributes(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return value;
|
||||
}
|
||||
public String toString() {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Slf4j
|
||||
public class DownloadJobWrapper implements Runnable {
|
||||
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final ConnectionService connectionService;
|
||||
private final ILogEventRepository eventRepository;
|
||||
|
||||
private final DownloadJob downloadJob;
|
||||
|
||||
public DownloadJobWrapper(final DownloadJob downloadJob) {
|
||||
downloadService = SpringContext.getBean(DownloadService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
connectionService = SpringContext.getBean(ConnectionService.class);
|
||||
eventRepository = SpringContext.getBean(ILogEventRepository.class);
|
||||
|
||||
this.downloadJob = downloadJob;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
Failsafe.with(connectionService.getRetryPolicy())
|
||||
.onFailure(
|
||||
e -> {
|
||||
try {
|
||||
LogEvent logEvent =
|
||||
new LogEvent(
|
||||
LogEvent.Type.DOWNLOAD,
|
||||
LogEvent.Status.ERROR,
|
||||
LocalDateTime.now(),
|
||||
String.format(
|
||||
"Failed to download %s\n %s",
|
||||
downloadJob.getImage().getUrl(),
|
||||
Utils.throwableToString(e.getFailure())));
|
||||
eventRepository.save(logEvent);
|
||||
} catch (Exception exp) {
|
||||
log.error("Failed to save event", exp);
|
||||
}
|
||||
log.error(
|
||||
String.format(
|
||||
"Failed to download %s after %d tries",
|
||||
downloadJob.getImage().getUrl(), e.getAttemptCount()),
|
||||
e.getFailure());
|
||||
downloadJob.getImage().setStatus(Status.ERROR);
|
||||
dataService.updateImageStatus(
|
||||
downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
|
||||
})
|
||||
.onComplete(
|
||||
e -> {
|
||||
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
|
||||
downloadService.afterJobFinish(downloadJob);
|
||||
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
|
||||
})
|
||||
.run(downloadJob);
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.Utils;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Slf4j
|
||||
public class DownloadRunnable implements Runnable {
|
||||
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final ConnectionService connectionService;
|
||||
private final IEventRepository eventRepository;
|
||||
|
||||
private final DownloadJob downloadJob;
|
||||
|
||||
public DownloadRunnable(final DownloadJob downloadJob) {
|
||||
downloadService = SpringContext.getBean(DownloadService.class);
|
||||
dataService = SpringContext.getBean(DataService.class);
|
||||
connectionService = SpringContext.getBean(ConnectionService.class);
|
||||
eventRepository = SpringContext.getBean(IEventRepository.class);
|
||||
|
||||
this.downloadJob = downloadJob;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
Failsafe.with(connectionService.getRetryPolicy())
|
||||
.onFailure(e -> {
|
||||
try {
|
||||
Event event = new Event(Event.Type.DOWNLOAD, Event.Status.ERROR, LocalDateTime.now(), String.format("Failed to download %s\n %s", downloadJob.getImage().getUrl(), Utils.throwableToString(e.getFailure())));
|
||||
eventRepository.save(event);
|
||||
} catch (Exception exp) {
|
||||
log.error("Failed to save event", exp);
|
||||
}
|
||||
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
|
||||
downloadJob.getImage().setStatus(Status.ERROR);
|
||||
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
|
||||
})
|
||||
.onComplete(e -> {
|
||||
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
|
||||
downloadService.afterJobFinish(downloadJob);
|
||||
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
|
||||
}).run(downloadJob);
|
||||
}
|
||||
}
|
||||
@@ -9,215 +9,269 @@ import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.PostService;
|
||||
import tn.mnlr.vripper.services.MetadataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DownloadService {
|
||||
|
||||
private static final List<Status> FINISHED = Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
|
||||
private final int MAX_POOL_SIZE = 12;
|
||||
private final int MAX_POOL_SIZE = 12;
|
||||
|
||||
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
|
||||
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
|
||||
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
|
||||
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
|
||||
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
|
||||
private final List<DownloadJob> running = new ArrayList<>();
|
||||
private final List<DownloadJob> pending = new ArrayList<>();
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final SettingsService settings;
|
||||
private final DataService dataService;
|
||||
private final PostService postService;
|
||||
private final SettingsService settingsService;
|
||||
private final DataService dataService;
|
||||
private final MetadataService metadataService;
|
||||
|
||||
private boolean pauseQ = false;
|
||||
private Thread executionThread;
|
||||
private Thread pollThread;
|
||||
private final List<Host> hosts;
|
||||
|
||||
@Autowired
|
||||
public DownloadService(PendingQueue pendingQueue, SettingsService settings, DataService dataService, PostService postService) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
this.settings = settings;
|
||||
this.dataService = dataService;
|
||||
this.postService = postService;
|
||||
private Thread pollThread;
|
||||
|
||||
@Autowired
|
||||
public DownloadService(
|
||||
SettingsService settingsService,
|
||||
DataService dataService,
|
||||
MetadataService metadataService,
|
||||
List<Host> hosts) {
|
||||
this.settingsService = settingsService;
|
||||
this.dataService = dataService;
|
||||
this.metadataService = metadataService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
pollThread = new Thread(this::start, "Polling thread");
|
||||
pollThread.start();
|
||||
}
|
||||
|
||||
public void destroy() throws Exception {
|
||||
log.info("Shutting down ExecutionService");
|
||||
pollThread.interrupt();
|
||||
executor.shutdown();
|
||||
dataService
|
||||
.findAllPosts()
|
||||
.forEach(
|
||||
p -> {
|
||||
log.debug(String.format("Stopping download jobs for %s", p));
|
||||
this.stopRunning(p.getPostId());
|
||||
});
|
||||
executor.awaitTermination(5, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private synchronized void stopRunning(@NonNull String postId) {
|
||||
List<DownloadJob> stopping = new ArrayList<>();
|
||||
Iterator<DownloadJob> iterator = running.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob downloadJob = iterator.next();
|
||||
if (postId.equals(downloadJob.getPost().getPostId())) {
|
||||
downloadJob.stop();
|
||||
iterator.remove();
|
||||
stopping.add(downloadJob);
|
||||
}
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
executionThread = new Thread(this::start, "Executor thread");
|
||||
pollThread = new Thread(this::poll, "Polling thread");
|
||||
pollThread.start();
|
||||
executionThread.start();
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(DownloadJob::isFinished);
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void destroy() throws Exception {
|
||||
log.info("Shutting down ExecutionService");
|
||||
executionThread.interrupt();
|
||||
pollThread.interrupt();
|
||||
executor.shutdown();
|
||||
dataService.findAllPosts().forEach(p -> {
|
||||
log.debug(String.format("Stopping download jobs for %s", p));
|
||||
this.stopRunning(p.getPostId());
|
||||
public void stopAll(List<String> postIds) {
|
||||
stop(
|
||||
Objects.requireNonNullElseGet(
|
||||
postIds,
|
||||
() ->
|
||||
dataService.findAllPosts().stream()
|
||||
.map(Post::getPostId)
|
||||
.collect(Collectors.toList())));
|
||||
}
|
||||
|
||||
public void restartAll(List<String> posIds) {
|
||||
restart(
|
||||
Objects.requireNonNullElseGet(
|
||||
posIds,
|
||||
() ->
|
||||
dataService.findAllPosts().stream()
|
||||
.map(Post::getPostId)
|
||||
.collect(Collectors.toList())));
|
||||
}
|
||||
|
||||
private void restart(@NonNull List<String> postIds) {
|
||||
Map<Post, Collection<Image>> data = new HashMap<>();
|
||||
for (String postId : postIds) {
|
||||
if (isPending(postId)) {
|
||||
log.warn(
|
||||
String.format("Cannot restart, jobs are currently running for post id %s", postIds));
|
||||
continue;
|
||||
}
|
||||
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
|
||||
if (images.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postIds));
|
||||
data.put(post, images);
|
||||
}
|
||||
enqueue(data);
|
||||
}
|
||||
|
||||
private synchronized boolean isPending(String postId) {
|
||||
return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
|
||||
}
|
||||
|
||||
private synchronized boolean isRunning(String postId) {
|
||||
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
|
||||
}
|
||||
|
||||
private synchronized void stop(List<String> postIds) {
|
||||
|
||||
for (String postId : postIds) {
|
||||
final Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (post == null) {
|
||||
continue;
|
||||
}
|
||||
pending.removeIf(p -> p.getPost().equals(post));
|
||||
stopRunning(postId);
|
||||
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
|
||||
dataService.finishPost(post);
|
||||
}
|
||||
metadataService.stopFetchingMetadata(postIds);
|
||||
}
|
||||
|
||||
private boolean canRun(Host host) {
|
||||
boolean canRun;
|
||||
int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
canRun =
|
||||
threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
|
||||
&& (settingsService.getSettings().getMaxTotalThreads() == 0
|
||||
? totalRunning < MAX_POOL_SIZE
|
||||
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
|
||||
if (canRun) {
|
||||
threadCount.get(host).incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private Map<Host, Integer> candidateCount() {
|
||||
HashMap<Host, Integer> map = new HashMap<>();
|
||||
hosts.forEach(
|
||||
h -> {
|
||||
AtomicInteger count = threadCount.get(h);
|
||||
if (count == null) {
|
||||
count = new AtomicInteger(0);
|
||||
threadCount.put(h, count);
|
||||
}
|
||||
map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
|
||||
});
|
||||
executor.awaitTermination(5, TimeUnit.SECONDS);
|
||||
return map;
|
||||
}
|
||||
|
||||
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount, int max) {
|
||||
int i = 0;
|
||||
List<DownloadJob> candidates = new ArrayList<>();
|
||||
for (DownloadJob downloadJob : pending) {
|
||||
Host host = downloadJob.getImage().getHost();
|
||||
Integer maxPerHost = candidateCount.get(host);
|
||||
if (maxPerHost > 0 && i < max) {
|
||||
candidates.add(downloadJob);
|
||||
candidateCount.put(host, maxPerHost + 1);
|
||||
i++;
|
||||
}
|
||||
if (i >= max) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
private void stopRunning(@NonNull String postId) {
|
||||
List<DownloadJob> stopping = new ArrayList<>();
|
||||
Iterator<DownloadJob> iterator = executing.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DownloadJob downloadJob = iterator.next();
|
||||
if (postId.equals(downloadJob.getPost().getPostId())) {
|
||||
downloadJob.stop();
|
||||
iterator.remove();
|
||||
stopping.add(downloadJob);
|
||||
}
|
||||
}
|
||||
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(DownloadJob::isFinished);
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
public void enqueue(Map<Post, Collection<Image>> images) {
|
||||
synchronized (this) {
|
||||
for (Map.Entry<Post, Collection<Image>> entry : images.entrySet()) {
|
||||
entry.getKey().setStatus(Status.PENDING);
|
||||
dataService.updatePostStatus(entry.getKey().getStatus(), entry.getKey().getId());
|
||||
for (Image image : entry.getValue()) {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob =
|
||||
new DownloadJob(
|
||||
entry.getKey(), image, (Settings) settingsService.getSettings().clone());
|
||||
pending.add(downloadJob);
|
||||
}
|
||||
}
|
||||
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
public void stopAll(List<String> posIds) {
|
||||
if (posIds != null) {
|
||||
posIds.forEach(this::stop);
|
||||
} else {
|
||||
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
|
||||
}
|
||||
}
|
||||
|
||||
public void restartAll(List<String> posIds) {
|
||||
if (posIds != null) {
|
||||
posIds.forEach(this::restart);
|
||||
} else {
|
||||
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
|
||||
}
|
||||
}
|
||||
|
||||
private void restart(@NonNull String postId) {
|
||||
if (isPending(postId)) {
|
||||
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
|
||||
return;
|
||||
}
|
||||
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
|
||||
if (images.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
post.setStatus(Status.PENDING);
|
||||
dataService.updatePostStatus(post.getStatus(), post.getId());
|
||||
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
|
||||
for (Image image : images) {
|
||||
try {
|
||||
pendingQueue.put(post, image);
|
||||
} catch (InterruptedException e) {
|
||||
log.warn("Thread was interrupted", e);
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isPending(String postId) {
|
||||
return pendingQueue.isPending(postId);
|
||||
}
|
||||
|
||||
private void stop(String postId) {
|
||||
try {
|
||||
pauseQ = true;
|
||||
final Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (post == null) {
|
||||
return;
|
||||
}
|
||||
if (FINISHED.contains(post.getStatus())) {
|
||||
return;
|
||||
}
|
||||
pendingQueue.stop(post);
|
||||
stopRunning(postId);
|
||||
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
|
||||
dataService.finishPost(post);
|
||||
postService.stopFetchingMetadata(post);
|
||||
} finally {
|
||||
pauseQ = false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean canRun(Host host) {
|
||||
boolean canRun;
|
||||
AtomicInteger count = threadCount.get(host);
|
||||
if (count == null) {
|
||||
threadCount.put(host, new AtomicInteger(0));
|
||||
}
|
||||
canRun = threadCount.get(host).get() < settings.getSettings().getMaxThreads() && (settings.getSettings().getMaxTotalThreads() == 0 ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE : threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getSettings().getMaxTotalThreads());
|
||||
if (canRun && !pauseQ) {
|
||||
threadCount.get(host).incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void poll() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
List<DownloadJob> peek = pendingQueue.peek();
|
||||
for (DownloadJob downloadJob : peek) {
|
||||
if (canRun(downloadJob.getImage().getHost())) {
|
||||
executionQueue.offer(downloadJob);
|
||||
pendingQueue.remove(downloadJob);
|
||||
}
|
||||
private void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
synchronized (this) {
|
||||
List<DownloadJob> accepted = new ArrayList<>();
|
||||
List<DownloadJob> candidates = getCandidates(candidateCount(), MAX_POOL_SIZE);
|
||||
candidates.forEach(
|
||||
c -> {
|
||||
if (canRun(c.getImage().getHost())) {
|
||||
accepted.add(c);
|
||||
}
|
||||
synchronized (threadCount) {
|
||||
threadCount.wait(2_000);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
});
|
||||
pending.removeAll(accepted);
|
||||
accepted.forEach(this::push);
|
||||
accepted.clear();
|
||||
this.wait();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void start() {
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
push(executionQueue.take());
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
log.error("Execution Service failed", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
private synchronized void push(DownloadJob downloadJob) {
|
||||
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
|
||||
executor.execute(new DownloadJobWrapper(downloadJob));
|
||||
running.add(downloadJob);
|
||||
}
|
||||
|
||||
private void push(DownloadJob downloadJob) {
|
||||
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
|
||||
executor.execute(new DownloadRunnable(downloadJob));
|
||||
executing.add(downloadJob);
|
||||
public void afterJobFinish(DownloadJob downloadJob) {
|
||||
synchronized (this) {
|
||||
running.remove(downloadJob);
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
if (!isPending(downloadJob.getPost().getPostId())
|
||||
&& !isRunning(downloadJob.getPost().getPostId())) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(DownloadJob downloadJob) {
|
||||
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
|
||||
if (count == 0) {
|
||||
dataService.finishPost(downloadJob.getPost());
|
||||
}
|
||||
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
|
||||
executing.remove(downloadJob);
|
||||
synchronized (threadCount) {
|
||||
threadCount.notify();
|
||||
}
|
||||
}
|
||||
public int pendingCount() {
|
||||
return pending.size();
|
||||
}
|
||||
|
||||
public int runningCount() {
|
||||
return executing.size();
|
||||
}
|
||||
public int runningCount() {
|
||||
return running.size();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package tn.mnlr.vripper.download;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.DataService;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.BlockingDeque;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class PendingQueue {
|
||||
|
||||
private final DataService dataService;
|
||||
private final SettingsService settingsService;
|
||||
private final List<Host> hosts;
|
||||
|
||||
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
|
||||
|
||||
@Autowired
|
||||
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
|
||||
this.dataService = dataService;
|
||||
this.settingsService = settingsService;
|
||||
this.hosts = hosts;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
|
||||
}
|
||||
|
||||
public void put(Post post, Image image) throws InterruptedException {
|
||||
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
|
||||
image.init();
|
||||
dataService.updateImageStatus(image.getStatus(), image.getId());
|
||||
dataService.updateImageCurrent(image.getCurrent(), image.getId());
|
||||
DownloadJob downloadJob = new DownloadJob(post, image);
|
||||
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
|
||||
checkKey(post.getPostId());
|
||||
toBeExecuted.get(post.getPostId()).incrementAndGet();
|
||||
}
|
||||
|
||||
private synchronized void checkKey(String postId) {
|
||||
if (!toBeExecuted.containsKey(postId)) {
|
||||
toBeExecuted.put(postId, new AtomicInteger(0));
|
||||
}
|
||||
}
|
||||
|
||||
public void remove(final DownloadJob downloadJob) {
|
||||
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
|
||||
}
|
||||
|
||||
public List<DownloadJob> peek() {
|
||||
List<DownloadJob> downloadJobs = new ArrayList<>();
|
||||
if (hosts.size() == 0) {
|
||||
return downloadJobs;
|
||||
}
|
||||
for (Host host : hosts) {
|
||||
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
|
||||
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
|
||||
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
|
||||
if (downloadJob != null) {
|
||||
downloadJobs.add(downloadJob);
|
||||
}
|
||||
}
|
||||
}
|
||||
return downloadJobs;
|
||||
}
|
||||
|
||||
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
|
||||
for (Image image : images) {
|
||||
put(post, image);
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
|
||||
}
|
||||
|
||||
public void stop(Post post) {
|
||||
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
|
||||
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
|
||||
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
|
||||
entry.getValue().removeIf(predicate);
|
||||
decrement(post.getPostId());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isPending(String postId) {
|
||||
return toBeExecuted.containsKey(postId);
|
||||
}
|
||||
|
||||
public synchronized int decrement(String postId) {
|
||||
AtomicInteger counter = toBeExecuted.get(postId);
|
||||
if (counter == null) {
|
||||
return 0;
|
||||
}
|
||||
int count = counter.decrementAndGet();
|
||||
if (count == 0) {
|
||||
toBeExecuted.remove(postId);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public class Event<T> {
|
||||
|
||||
private final T data;
|
||||
private final Kind kind;
|
||||
|
||||
private Event(Kind kind, T data) {
|
||||
this.data = data;
|
||||
this.kind = kind;
|
||||
}
|
||||
|
||||
public static <T> Event<T> wrap(Kind kind, T data) {
|
||||
return new Event<>(kind, data);
|
||||
}
|
||||
|
||||
public enum Kind {
|
||||
POST_UPDATE,
|
||||
POST_REMOVE,
|
||||
IMAGE_UPDATE,
|
||||
METADATA_UPDATE,
|
||||
QUEUED_UPDATE,
|
||||
QUEUED_REMOVE,
|
||||
LOG_EVENT_UPDATE,
|
||||
LOG_EVENT_REMOVE,
|
||||
VG_USER,
|
||||
GLOBAL_STATE,
|
||||
BYTES_PER_SECOND,
|
||||
SETTINGS_UPDATE
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Service
|
||||
public class EventBus {
|
||||
|
||||
public static final Sinks.EmitFailureHandler RETRY = (signalType, emitResult) -> true;
|
||||
private final Sinks.Many<Event<?>> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
public void publishEvent(Event<?> event) {
|
||||
sink.emitNext(event, RETRY);
|
||||
}
|
||||
|
||||
public Flux<Event<?>> flux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(RETRY);
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class EventRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public EventRemoveEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class EventUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public EventUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class ImageUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public ImageUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class MetadataUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long postIdRef;
|
||||
|
||||
public MetadataUpdateEvent(Object source, Long postIdRef) {
|
||||
super(source);
|
||||
this.postIdRef = postIdRef;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class PostRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final String postId;
|
||||
|
||||
public PostRemoveEvent(Object source, String postId) {
|
||||
super(source);
|
||||
this.postId = postId;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class PostUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public PostUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class QueuedRemoveEvent extends ApplicationEvent {
|
||||
|
||||
private final String threadId;
|
||||
|
||||
public QueuedRemoveEvent(Object source, String threadId) {
|
||||
super(source);
|
||||
this.threadId = threadId;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package tn.mnlr.vripper.event;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
@Getter
|
||||
public class QueuedUpdateEvent extends ApplicationEvent {
|
||||
|
||||
private final Long id;
|
||||
|
||||
public QueuedUpdateEvent(Object source, Long id) {
|
||||
super(source);
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,11 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class DownloadException extends Exception {
|
||||
|
||||
public DownloadException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public DownloadException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DownloadException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public DownloadException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class HostException extends Exception {
|
||||
public HostException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public HostException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
|
||||
public HostException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public HostException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public HostException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
public HostException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class HtmlProcessorException extends Exception {
|
||||
|
||||
public HtmlProcessorException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public HtmlProcessorException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,15 +2,15 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class PostParseException extends Exception {
|
||||
|
||||
public PostParseException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public PostParseException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public PostParseException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
public PostParseException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
|
||||
public PostParseException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public PostParseException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,15 +2,15 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class QueueException extends Exception {
|
||||
|
||||
public QueueException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public QueueException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public QueueException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
public QueueException(String message, Throwable e) {
|
||||
super(message, e);
|
||||
}
|
||||
|
||||
public QueueException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public QueueException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import java.io.IOException;
|
||||
|
||||
public class RenameException extends Exception {
|
||||
|
||||
public RenameException(String message, IOException e) {
|
||||
super(message, e);
|
||||
}
|
||||
public RenameException(String message, IOException e) {
|
||||
super(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
|
||||
|
||||
public class ValidationException extends Exception {
|
||||
|
||||
public ValidationException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public ValidationException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class VripperException extends Exception {
|
||||
public VripperException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public VripperException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public VripperException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public VripperException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package tn.mnlr.vripper.exception;
|
||||
|
||||
public class XpathException extends Exception {
|
||||
public XpathException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
public XpathException(Throwable e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,90 +27,96 @@ import java.util.List;
|
||||
@Slf4j
|
||||
public class AcidimgHost extends Host {
|
||||
|
||||
private static final String host = "acidimg.cc";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
private static final String host = "acidimg.cc";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
|
||||
private final ConnectionService cm;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final ConnectionService cm;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
|
||||
@Autowired
|
||||
public AcidimgHost(
|
||||
ConnectionService cm,
|
||||
HostService hostService,
|
||||
XpathService xpathService,
|
||||
HtmlProcessorService htmlProcessorService) {
|
||||
this.cm = cm;
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public AcidimgHost(ConnectionService cm, HostService hostService, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
|
||||
this.cm = cm;
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
if (contDiv != null) {
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url, context);
|
||||
httpPost.addHeader("Referer", url);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
log.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response =
|
||||
(CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
log.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url, context);
|
||||
httpPost.addHeader("Referer", url);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
log.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
log.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Cannot find the image node");
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Cannot find the image node");
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,33 +10,34 @@ import java.util.Objects;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
abstract public class Host {
|
||||
public abstract class Host {
|
||||
|
||||
abstract public String getHost();
|
||||
public abstract String getHost();
|
||||
|
||||
abstract public String getLookup();
|
||||
public abstract String getLookup();
|
||||
|
||||
public boolean isSupported(String url) {
|
||||
return url.contains(getLookup());
|
||||
}
|
||||
public boolean isSupported(String url) {
|
||||
return url.contains(getLookup());
|
||||
}
|
||||
|
||||
public abstract HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException;
|
||||
public abstract HostService.NameUrl getNameAndUrl(
|
||||
final String url, final HttpClientContext context) throws HostException;
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Host host = (Host) o;
|
||||
return Objects.equals(getHost(), host.getHost());
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Host host = (Host) o;
|
||||
return Objects.equals(getHost(), host.getHost());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(getHost());
|
||||
}
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(getHost());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getHost();
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return getHost();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,103 +1,96 @@
|
||||
package tn.mnlr.vripper.host;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.protocol.HttpClientContext;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.HostException;
|
||||
import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
import tn.mnlr.vripper.exception.XpathException;
|
||||
import tn.mnlr.vripper.services.ConnectionService;
|
||||
import tn.mnlr.vripper.services.HostService;
|
||||
import tn.mnlr.vripper.services.HtmlProcessorService;
|
||||
import tn.mnlr.vripper.services.XpathService;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ImageBamHost extends Host {
|
||||
|
||||
private static final String host = "imagebam.com";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
|
||||
private static final String IMG_XPATH = "//img[@class='image']";
|
||||
private static final String host = "imagebam.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class,'main-image')]";
|
||||
private static final String CONTINUE_XPATH = "//*[contains(text(), 'Continue')]";
|
||||
|
||||
private final HostService hostService;
|
||||
private final ConnectionService cm;
|
||||
private final XpathService xpathService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImageBamHost(HostService hostService, ConnectionService cm, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
|
||||
this.hostService = hostService;
|
||||
this.cm = cm;
|
||||
this.xpathService = xpathService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
@Autowired
|
||||
public ImageBamHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
HostService.Response response = hostService.getResponse(url, context);
|
||||
Document doc = response.getDocument();
|
||||
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, url));
|
||||
if (xpathService.getAsNode(doc, CONTINUE_XPATH) != null) {
|
||||
// Button detected. No need to actually click it, just make the call again.
|
||||
response = hostService.getResponse(url, context);
|
||||
doc = response.getDocument();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
if (imgNode == null) {
|
||||
throw new HostException(String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, url));
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle =
|
||||
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
|
||||
.map(e -> e.getTextContent().trim())
|
||||
.orElse("");
|
||||
String imgUrl =
|
||||
Optional.ofNullable(imgNode.getAttributes().getNamedItem("src"))
|
||||
.map(e -> e.getTextContent().trim())
|
||||
.orElse("");
|
||||
String defaultName = UUID.randomUUID().toString();
|
||||
|
||||
HostService.Response response = hostService.getResponse(url, context);
|
||||
Document doc = response.getDocument();
|
||||
int index = imgUrl.lastIndexOf('/');
|
||||
if (index != -1 && index < imgUrl.length()) {
|
||||
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
|
||||
}
|
||||
|
||||
Node contDiv;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (contDiv != null) {
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url, context);
|
||||
httpGet.addHeader("Referer", url);
|
||||
log.debug(String.format("Requesting %s", httpGet));
|
||||
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
String s = EntityUtils.toString(res.getEntity());
|
||||
log.debug(String.format("%s response is:%n%s", httpGet, s));
|
||||
log.debug(String.format("Cleaning response for %s", httpGet));
|
||||
doc = htmlProcessorService.clean(s);
|
||||
EntityUtils.consumeQuietly(res.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? defaultName : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,49 +17,54 @@ import java.util.Optional;
|
||||
@Slf4j
|
||||
public class ImageTwistHost extends Host {
|
||||
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String host = "imagetwist.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String host = "imagetwist.com";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImageTwistHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public ImageTwistHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle =
|
||||
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
|
||||
.map(Node::getTextContent)
|
||||
.map(String::trim)
|
||||
.orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,68 +15,70 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class ImageVenueHost extends Host {
|
||||
|
||||
private static final String host = "imagevenue.com";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
|
||||
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
|
||||
private static final String host = "imagevenue.com";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
|
||||
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImageVenueHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public ImageVenueHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
|
||||
// Button detected. No need to actually click it, just make the call again.
|
||||
resp = hostService.getResponse(url, context);
|
||||
doc = resp.getDocument();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Failed to locate image");
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
|
||||
//Button detected. No need to actually click it, just make the call again.
|
||||
resp = hostService.getResponse(url, context);
|
||||
doc = resp.getDocument();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Failed to locate image");
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,56 +15,57 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class ImageZillaHost extends Host {
|
||||
|
||||
private static final String host = "imagezilla.net";
|
||||
private static final String lookup = "imagezilla.net/show";
|
||||
private static final String IMG_XPATH = "//img[@id='photo']";
|
||||
private static final String host = "imagezilla.net";
|
||||
private static final String lookup = "imagezilla.net/show";
|
||||
private static final String IMG_XPATH = "//img[@id='photo']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImageZillaHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public ImageZillaHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
|
||||
log.debug(String.format("Resolving name for %s", url));
|
||||
if (titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
title = null;
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
if (title == null || title.isEmpty()) {
|
||||
title = hostService.getDefaultImageName(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
|
||||
log.debug(String.format("Resolving name for %s", url));
|
||||
if (titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
title = null;
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (title == null || title.isEmpty()) {
|
||||
title = hostService.getDefaultImageName(url);
|
||||
}
|
||||
|
||||
try {
|
||||
return new HostService.NameUrl(title, url.replace("show", "images"));
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
return new HostService.NameUrl(title, url.replace("show", "images"));
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,51 +15,53 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class ImgSpiceHost extends Host {
|
||||
|
||||
private static final String host = "imgspice.com";
|
||||
private static final String IMG_XPATH = "//img[@id='imgpreview']";
|
||||
private static final String host = "imgspice.com";
|
||||
private static final String IMG_XPATH = "//img[@id='imgpreview']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,50 +15,50 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class ImgboxHost extends Host {
|
||||
|
||||
private static final String host = "imgbox.com";
|
||||
private static final String IMG_XPATH = "//img[@id='img']";
|
||||
private static final String host = "imgbox.com";
|
||||
private static final String IMG_XPATH = "//img[@id='img']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ImgboxHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public ImgboxHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,94 +29,101 @@ import java.util.List;
|
||||
@Slf4j
|
||||
public class ImxHost extends Host {
|
||||
|
||||
private static final String host = "imx.to";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
private static final String host = "imx.to";
|
||||
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
|
||||
private static final String IMG_XPATH = "//img[@class='centred']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final ConnectionService cm;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final ConnectionService cm;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
|
||||
@Autowired
|
||||
public ImxHost(HostService hostService, XpathService xpathService, ConnectionService cm, HtmlProcessorService htmlProcessorService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
this.cm = cm;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
@Autowired
|
||||
public ImxHost(
|
||||
HostService hostService,
|
||||
XpathService xpathService,
|
||||
ConnectionService cm,
|
||||
HtmlProcessorService htmlProcessorService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
this.cm = cm;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node contDiv;
|
||||
String value = null;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
if (contDiv == null) {
|
||||
throw new HostException(CONTINUE_BUTTON_XPATH + " cannot be found");
|
||||
}
|
||||
Node node = contDiv.getAttributes().getNamedItem("value");
|
||||
if (node != null) {
|
||||
value = node.getTextContent();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
if (value == null) {
|
||||
throw new HostException("Failed to obtain value attribute from continue input");
|
||||
}
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url, context);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", value));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
log.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response =
|
||||
(CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
log.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node contDiv;
|
||||
String value = null;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
|
||||
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
|
||||
if (contDiv == null) {
|
||||
throw new HostException(CONTINUE_BUTTON_XPATH + " cannot be found");
|
||||
}
|
||||
Node node = contDiv.getAttributes().getNamedItem("value");
|
||||
if (node != null) {
|
||||
value = node.getTextContent();
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (value == null) {
|
||||
throw new HostException("Failed to obtain value attribute from continue input");
|
||||
}
|
||||
log.debug(String.format("Click button found for %s", url));
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpPost httpPost = cm.buildHttpPost(url, context);
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("imgContinue", value));
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(params));
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
log.debug(String.format("Requesting %s", httpPost));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
|
||||
log.debug(String.format("Cleaning response for %s", httpPost));
|
||||
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException | HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,61 +15,61 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class PimpandhostHost extends Host {
|
||||
|
||||
private static final String host = "pimpandhost.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
|
||||
private static final String host = "pimpandhost.com";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PimpandhostHost(HostService hostService, XpathService xpathService) {
|
||||
super();
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public PimpandhostHost(HostService hostService, XpathService xpathService) {
|
||||
super();
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url;
|
||||
try {
|
||||
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url;
|
||||
try {
|
||||
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
|
||||
} catch (Exception e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle,
|
||||
imgUrl
|
||||
);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl =
|
||||
"https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,55 +15,55 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class PixRouteHost extends Host {
|
||||
|
||||
private static final String host = "pixroute.com";
|
||||
private static final String IMG_XPATH = "//img[@id='imgpreview']";
|
||||
private static final String host = "pixroute.com";
|
||||
private static final String IMG_XPATH = "//img[@id='imgpreview']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PixRouteHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public PixRouteHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Failed to locate image");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (imgNode == null) {
|
||||
throw new HostException("Failed to locate image");
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
|
||||
imgNode.getAttributes().getNamedItem("src").getTextContent().trim()
|
||||
);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
|
||||
imgNode.getAttributes().getNamedItem("src").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,50 +15,51 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class PixhostHost extends Host {
|
||||
|
||||
private static final String host = "pixhost.to";
|
||||
private static final String lookup = "pixhost.to/show";
|
||||
private static final String IMG_XPATH = "//img[@id='image']";
|
||||
private static final String host = "pixhost.to";
|
||||
private static final String lookup = "pixhost.to/show";
|
||||
private static final String IMG_XPATH = "//img[@id='image']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PixhostHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public PixhostHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return lookup;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,56 +15,58 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class PixxxelsHost extends Host {
|
||||
|
||||
private static final String host = "pixxxels.cc";
|
||||
private static final String IMG_XPATH = "//*[@id='download']";
|
||||
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
|
||||
private static final String host = "pixxxels.cc";
|
||||
private static final String IMG_XPATH = "//*[@id='download']";
|
||||
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PixxxelsHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public PixxxelsHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode, titleNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
HostService.Response resp = hostService.getResponse(url, context);
|
||||
Document doc = resp.getDocument();
|
||||
|
||||
Node imgNode, titleNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = titleNode.getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = titleNode.getTextContent().trim();
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,53 +17,58 @@ import java.util.Optional;
|
||||
@Slf4j
|
||||
public class PostImgHost extends Host {
|
||||
|
||||
private static final String host = "postimg.cc";
|
||||
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
|
||||
private static final String IMG_XPATH = "//a[@id='download']";
|
||||
private static final String host = "postimg.cc";
|
||||
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
|
||||
private static final String IMG_XPATH = "//a[@id='download']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public PostImgHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public PostImgHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node urlNode, titleNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
|
||||
|
||||
String url = _url.replace("http://", "https://");
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node urlNode, titleNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> hostService.getDefaultImageName(url));
|
||||
|
||||
return new HostService.NameUrl(imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle =
|
||||
Optional.ofNullable(titleNode)
|
||||
.map(node -> node.getTextContent().trim())
|
||||
.orElseGet(() -> hostService.getDefaultImageName(url));
|
||||
|
||||
return new HostService.NameUrl(
|
||||
imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,57 +15,59 @@ import tn.mnlr.vripper.services.XpathService;
|
||||
@Slf4j
|
||||
public class TurboImageHost extends Host {
|
||||
|
||||
private static final String host = "turboimagehost.com";
|
||||
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
|
||||
private static final String IMG_XPATH = "//img[@id='imageid']";
|
||||
private static final String host = "turboimagehost.com";
|
||||
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
|
||||
private static final String IMG_XPATH = "//img[@id='imageid']";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public TurboImageHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public TurboImageHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
log.debug(String.format("Resolving name for %s", url));
|
||||
if (titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
title = null;
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
if (title == null || title.isEmpty()) {
|
||||
title = hostService.getDefaultImageName(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
String title;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
|
||||
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
|
||||
log.debug(String.format("Resolving name for %s", url));
|
||||
if (titleNode != null) {
|
||||
title = titleNode.getTextContent().trim();
|
||||
} else {
|
||||
title = null;
|
||||
}
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
if (title == null || title.isEmpty()) {
|
||||
title = hostService.getDefaultImageName(url);
|
||||
}
|
||||
|
||||
try {
|
||||
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
return new HostService.NameUrl(title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
return new HostService.NameUrl(
|
||||
title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,49 +17,54 @@ import java.util.Optional;
|
||||
@Slf4j
|
||||
public class ViprImHost extends Host {
|
||||
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String host = "vipr.im";
|
||||
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
|
||||
private static final String host = "vipr.im";
|
||||
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
private final HostService hostService;
|
||||
private final XpathService xpathService;
|
||||
|
||||
@Autowired
|
||||
public ViprImHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
@Autowired
|
||||
public ViprImHost(HostService hostService, XpathService xpathService) {
|
||||
this.hostService = hostService;
|
||||
this.xpathService = xpathService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLookup() {
|
||||
return host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
|
||||
|
||||
Document doc = hostService.getResponse(url, context).getDocument();
|
||||
|
||||
Node imgNode;
|
||||
try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
|
||||
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
|
||||
} catch (XpathException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url));
|
||||
String imgTitle =
|
||||
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
|
||||
.map(Node::getTextContent)
|
||||
.map(String::trim)
|
||||
.orElse(null);
|
||||
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
|
||||
|
||||
return new HostService.NameUrl(imgTitle, imgUrl);
|
||||
} catch (Exception e) {
|
||||
throw new HostException("Unexpected error occurred", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,49 +22,58 @@ import java.util.regex.Pattern;
|
||||
@EnableScheduling
|
||||
public class Management {
|
||||
|
||||
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||
private static final Pattern BACKUP_FILE_PATTERN = Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
|
||||
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||
private static final Pattern BACKUP_FILE_PATTERN =
|
||||
Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final String backupFolder;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final DownloadService downloadService;
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final String backupFolder;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final DownloadService downloadService;
|
||||
|
||||
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName, ThreadPoolService threadPoolService, DownloadService downloadService) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.downloadService = downloadService;
|
||||
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
|
||||
public Management(
|
||||
JdbcTemplate jdbcTemplate,
|
||||
@Value("${base.dir}") String baseDir,
|
||||
@Value("${base.dir.name}") String baseDirName,
|
||||
ThreadPoolService threadPoolService,
|
||||
DownloadService downloadService) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.downloadService = downloadService;
|
||||
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
|
||||
try {
|
||||
threadPoolService.destroy();
|
||||
downloadService.destroy();
|
||||
this.jdbcTemplate.execute("SHUTDOWN");
|
||||
} catch (Exception ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
@PostConstruct
|
||||
@Scheduled(cron = "0 0 0 ? * *")
|
||||
private void backup() {
|
||||
|
||||
try {
|
||||
threadPoolService.destroy();
|
||||
downloadService.destroy();
|
||||
this.jdbcTemplate.execute("SHUTDOWN");
|
||||
} catch (Exception ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
for (File file :
|
||||
Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[] {})) {
|
||||
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
|
||||
if (matcher.find()) {
|
||||
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
|
||||
if (localDate.isEqual(LocalDate.now())) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
@Scheduled(cron = "0 0 0 ? * *")
|
||||
private void backup() {
|
||||
|
||||
for (File file : Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[]{})) {
|
||||
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
|
||||
if (matcher.find()) {
|
||||
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
|
||||
if (localDate.isEqual(LocalDate.now())) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create a backup file
|
||||
File backupFile = new File(backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
|
||||
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
|
||||
}
|
||||
// create a backup file
|
||||
File backupFile =
|
||||
new File(
|
||||
backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
|
||||
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public class DateTimeSerializer extends StdSerializer<LocalDateTime> {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
|
||||
|
||||
protected DateTimeSerializer() {
|
||||
super(LocalDateTime.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
|
||||
throws IOException {
|
||||
gen.writeString(value.format(DATE_TIME_FORMATTER));
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class Event {
|
||||
|
||||
private Long id;
|
||||
private Type type;
|
||||
private Status status;
|
||||
@JsonSerialize(using = DateTimeSerializer.class)
|
||||
private LocalDateTime time;
|
||||
private String message;
|
||||
|
||||
public Event(Type type, Status status, LocalDateTime time, String message) {
|
||||
this.type = type;
|
||||
this.status = status;
|
||||
this.time = time;
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public enum Type {
|
||||
POST,
|
||||
QUEUED,
|
||||
THANKS,
|
||||
METADATA,
|
||||
SCAN,
|
||||
DOWNLOAD,
|
||||
|
||||
// Below are deprecated events
|
||||
METADATA_CACHE_MISS,
|
||||
QUEUED_CACHE_MISS
|
||||
}
|
||||
|
||||
public enum Status {
|
||||
PENDING, PROCESSING, DONE, ERROR
|
||||
}
|
||||
}
|
||||
|
||||
class DateTimeSerializer extends StdSerializer<LocalDateTime> {
|
||||
|
||||
private final static DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
|
||||
|
||||
protected DateTimeSerializer() {
|
||||
super(LocalDateTime.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider) throws IOException {
|
||||
gen.writeString(value.format(DATE_TIME_FORMATTER));
|
||||
}
|
||||
}
|
||||
@@ -16,55 +16,51 @@ import java.util.Objects;
|
||||
@NoArgsConstructor
|
||||
public class Image {
|
||||
|
||||
@JsonIgnore
|
||||
protected Long id;
|
||||
@JsonIgnore protected Long id;
|
||||
|
||||
@JsonIgnore
|
||||
private Host host;
|
||||
@JsonIgnore private Host host;
|
||||
|
||||
private String url;
|
||||
private String url;
|
||||
|
||||
private int index;
|
||||
private int index;
|
||||
|
||||
private long current = 0;
|
||||
private long current = 0;
|
||||
|
||||
private long total = 0;
|
||||
private long total = 0;
|
||||
|
||||
private Status status;
|
||||
private Status status;
|
||||
|
||||
private String postId;
|
||||
private String postId;
|
||||
|
||||
@JsonIgnore
|
||||
private Long postIdRef;
|
||||
@JsonIgnore private Long postIdRef;
|
||||
|
||||
public Image(String postId, String url, Host host, int index) {
|
||||
this.postId = postId;
|
||||
this.url = url;
|
||||
this.host = host;
|
||||
this.index = index;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
public Image(String postId, String url, Host host, int index) {
|
||||
this.postId = postId;
|
||||
this.url = url;
|
||||
this.host = host;
|
||||
this.index = index;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
|
||||
public void increase(int read) {
|
||||
current += read;
|
||||
}
|
||||
|
||||
public void increase(int read) {
|
||||
current += read;
|
||||
}
|
||||
public void init() {
|
||||
current = 0;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
|
||||
public void init() {
|
||||
current = 0;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Image image = (Image) o;
|
||||
return Objects.equals(url, image.url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Image image = (Image) o;
|
||||
return Objects.equals(url, image.url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(url);
|
||||
}
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(url);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class LogEvent {
|
||||
|
||||
private Long id;
|
||||
private Type type;
|
||||
private Status status;
|
||||
|
||||
@JsonSerialize(using = DateTimeSerializer.class)
|
||||
private LocalDateTime time;
|
||||
|
||||
private String message;
|
||||
|
||||
public LogEvent(Type type, Status status, LocalDateTime time, String message) {
|
||||
this.type = type;
|
||||
this.status = status;
|
||||
this.time = time;
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public enum Type {
|
||||
POST,
|
||||
QUEUED,
|
||||
THANKS,
|
||||
METADATA,
|
||||
SCAN,
|
||||
DOWNLOAD,
|
||||
|
||||
// Below are deprecated events
|
||||
METADATA_CACHE_MISS,
|
||||
QUEUED_CACHE_MISS
|
||||
}
|
||||
|
||||
public enum Status {
|
||||
PENDING,
|
||||
PROCESSING,
|
||||
DONE,
|
||||
ERROR
|
||||
}
|
||||
}
|
||||
@@ -12,20 +12,20 @@ import java.util.List;
|
||||
@NoArgsConstructor
|
||||
public class Metadata {
|
||||
|
||||
private Long postIdRef;
|
||||
private Long postIdRef;
|
||||
|
||||
private String postId;
|
||||
private String postId;
|
||||
|
||||
private String postedBy;
|
||||
private String postedBy;
|
||||
|
||||
private List<String> resolvedNames = Collections.emptyList();
|
||||
private List<String> resolvedNames = Collections.emptyList();
|
||||
|
||||
public static Metadata from(Metadata metadata) {
|
||||
Metadata copy = new Metadata();
|
||||
copy.postIdRef = metadata.postIdRef;
|
||||
copy.postId = metadata.postId;
|
||||
copy.postedBy = metadata.postedBy;
|
||||
copy.resolvedNames = List.copyOf(metadata.resolvedNames);
|
||||
return copy;
|
||||
}
|
||||
public static Metadata from(Metadata metadata) {
|
||||
Metadata copy = new Metadata();
|
||||
copy.postIdRef = metadata.postIdRef;
|
||||
copy.postId = metadata.postId;
|
||||
copy.postedBy = metadata.postedBy;
|
||||
copy.resolvedNames = List.copyOf(metadata.resolvedNames);
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package tn.mnlr.vripper.jpa.domain;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Collections;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
@@ -17,62 +19,72 @@ import java.util.Set;
|
||||
@NoArgsConstructor
|
||||
public class Post {
|
||||
|
||||
@JsonIgnore
|
||||
private Long id;
|
||||
@JsonIgnore private Long id;
|
||||
|
||||
private Status status;
|
||||
private Status status;
|
||||
|
||||
private String postId;
|
||||
private String postId;
|
||||
|
||||
private String threadTitle;
|
||||
private String threadTitle;
|
||||
|
||||
private String threadId;
|
||||
private String threadId;
|
||||
|
||||
private String title;
|
||||
private String title;
|
||||
|
||||
private String url;
|
||||
private String url;
|
||||
|
||||
private int done;
|
||||
private int done;
|
||||
|
||||
private int total;
|
||||
private int total;
|
||||
|
||||
private Set<String> hosts;
|
||||
private Set<String> hosts;
|
||||
|
||||
private String forum;
|
||||
private String forum;
|
||||
|
||||
@JsonIgnore
|
||||
private String securityToken;
|
||||
@JsonIgnore private String securityToken;
|
||||
|
||||
@JsonIgnore
|
||||
private String downloadDirectory;
|
||||
@JsonIgnore private String downloadDirectory;
|
||||
|
||||
private boolean thanked;
|
||||
private boolean thanked;
|
||||
|
||||
private Set<String> previews = Collections.emptySet();
|
||||
private Set<String> previews = Collections.emptySet();
|
||||
|
||||
private Metadata metadata;
|
||||
private Metadata metadata;
|
||||
|
||||
public Post(String title, String url, String postId, String threadId, String threadTitle, String forum, String securityToken) {
|
||||
this.title = title;
|
||||
this.url = url;
|
||||
this.postId = postId;
|
||||
this.forum = forum;
|
||||
this.threadId = threadId;
|
||||
this.threadTitle = threadTitle;
|
||||
this.securityToken = securityToken;
|
||||
status = Status.STOPPED;
|
||||
}
|
||||
private int rank = Integer.MAX_VALUE;
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Post post = (Post) o;
|
||||
return Objects.equals(postId, post.postId);
|
||||
}
|
||||
@JsonSerialize(using = DateTimeSerializer.class)
|
||||
private LocalDateTime addedOn;
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(postId);
|
||||
}
|
||||
public Post(
|
||||
String title,
|
||||
String url,
|
||||
String postId,
|
||||
String threadId,
|
||||
String threadTitle,
|
||||
String forum,
|
||||
String securityToken) {
|
||||
this.title = title;
|
||||
this.url = url;
|
||||
this.postId = postId;
|
||||
this.forum = forum;
|
||||
this.threadId = threadId;
|
||||
this.threadTitle = threadTitle;
|
||||
this.securityToken = securityToken;
|
||||
status = Status.STOPPED;
|
||||
addedOn = LocalDateTime.now();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Post post = (Post) o;
|
||||
return Objects.equals(postId, post.postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(postId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,44 +14,43 @@ import java.util.Objects;
|
||||
@ToString
|
||||
public class Queued {
|
||||
|
||||
@JsonIgnore
|
||||
private Long id;
|
||||
@JsonIgnore private Long id;
|
||||
|
||||
private String link;
|
||||
private String link;
|
||||
|
||||
private String threadId;
|
||||
private String threadId;
|
||||
|
||||
private String postId;
|
||||
private String postId;
|
||||
|
||||
private int total = 0;
|
||||
private int total = 0;
|
||||
|
||||
private boolean loading = true;
|
||||
private boolean loading = true;
|
||||
|
||||
public Queued(String link, String threadId, String postId) {
|
||||
this();
|
||||
this.link = link;
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
public Queued(String link, String threadId, String postId) {
|
||||
this();
|
||||
this.link = link;
|
||||
this.threadId = threadId;
|
||||
this.postId = postId;
|
||||
}
|
||||
|
||||
public void done() {
|
||||
this.loading = false;
|
||||
}
|
||||
public void done() {
|
||||
this.loading = false;
|
||||
}
|
||||
|
||||
public void increment() {
|
||||
this.total++;
|
||||
}
|
||||
public void increment() {
|
||||
this.total++;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Queued that = (Queued) o;
|
||||
return Objects.equals(threadId, that.threadId);
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Queued that = (Queued) o;
|
||||
return Objects.equals(threadId, that.threadId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(threadId);
|
||||
}
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(threadId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
package tn.mnlr.vripper.jpa.domain.enums;
|
||||
|
||||
public enum Status {
|
||||
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED, PARTIAL
|
||||
}
|
||||
PENDING,
|
||||
DOWNLOADING,
|
||||
COMPLETE,
|
||||
ERROR,
|
||||
STOPPED,
|
||||
PARTIAL
|
||||
}
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IEventRepository extends IRepository {
|
||||
|
||||
Event save(Event event);
|
||||
|
||||
Event update(Event event);
|
||||
|
||||
List<Event> findAll();
|
||||
|
||||
Optional<Event> findById(Long id);
|
||||
|
||||
void delete(Long id);
|
||||
|
||||
void deleteAll();
|
||||
}
|
||||
+11
-11
@@ -8,25 +8,25 @@ import java.util.Optional;
|
||||
|
||||
public interface IImageRepository extends IRepository {
|
||||
|
||||
Image save(Image image);
|
||||
Image save(Image image);
|
||||
|
||||
void deleteAllByPostId(String postId);
|
||||
void deleteAllByPostId(String postId);
|
||||
|
||||
List<Image> findByPostId(String postId);
|
||||
List<Image> findByPostId(String postId);
|
||||
|
||||
Integer countError();
|
||||
Integer countError();
|
||||
|
||||
List<Image> findByPostIdAndIsNotCompleted(String postId);
|
||||
List<Image> findByPostIdAndIsNotCompleted(String postId);
|
||||
|
||||
int stopByPostIdAndIsNotCompleted(String postId);
|
||||
int stopByPostIdAndIsNotCompleted(String postId);
|
||||
|
||||
List<Image> findByPostIdAndIsError(String postId);
|
||||
List<Image> findByPostIdAndIsError(String postId);
|
||||
|
||||
Optional<Image> findById(Long id);
|
||||
Optional<Image> findById(Long id);
|
||||
|
||||
int updateStatus(Status status, Long id);
|
||||
int updateStatus(Status status, Long id);
|
||||
|
||||
int updateCurrent(long current, Long id);
|
||||
int updateCurrent(long current, Long id);
|
||||
|
||||
int updateTotal(long total, Long id);
|
||||
int updateTotal(long total, Long id);
|
||||
}
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface ILogEventRepository extends IRepository {
|
||||
|
||||
LogEvent save(LogEvent logEvent);
|
||||
|
||||
LogEvent update(LogEvent logEvent);
|
||||
|
||||
List<LogEvent> findAll();
|
||||
|
||||
Optional<LogEvent> findById(Long id);
|
||||
|
||||
void delete(Long id);
|
||||
|
||||
void deleteAll();
|
||||
}
|
||||
+3
-3
@@ -6,9 +6,9 @@ import java.util.Optional;
|
||||
|
||||
public interface IMetadataRepository extends IRepository {
|
||||
|
||||
Metadata save(Metadata metadata);
|
||||
Metadata save(Metadata metadata);
|
||||
|
||||
Optional<Metadata> findByPostId(String postId);
|
||||
Optional<Metadata> findByPostId(String postId);
|
||||
|
||||
int deleteByPostId(String postId);
|
||||
int deleteByPostId(String postId);
|
||||
}
|
||||
|
||||
+15
-13
@@ -8,29 +8,31 @@ import java.util.Optional;
|
||||
|
||||
public interface IPostRepository extends IRepository {
|
||||
|
||||
Post save(Post post);
|
||||
Post save(Post post);
|
||||
|
||||
Optional<Post> findByPostId(String postId);
|
||||
Optional<Post> findByPostId(String postId);
|
||||
|
||||
Optional<Post> findById(Long id);
|
||||
Optional<Post> findById(Long id);
|
||||
|
||||
List<String> findCompleted();
|
||||
List<String> findCompleted();
|
||||
|
||||
List<Post> findAll();
|
||||
List<Post> findAll();
|
||||
|
||||
boolean existByPostId(String postId);
|
||||
boolean existByPostId(String postId);
|
||||
|
||||
int setDownloadingToStopped();
|
||||
int setDownloadingToStopped();
|
||||
|
||||
int deleteByPostId(String postId);
|
||||
int deleteByPostId(String postId);
|
||||
|
||||
int updateStatus(Status status, Long id);
|
||||
int updateStatus(Status status, Long id);
|
||||
|
||||
int updateDone(int done, Long id);
|
||||
int updateDone(int done, Long id);
|
||||
|
||||
int updateFolderName(String postFolderName, Long id);
|
||||
int updateFolderName(String postFolderName, Long id);
|
||||
|
||||
int updateTitle(String title, Long id);
|
||||
int updateTitle(String title, Long id);
|
||||
|
||||
int updateThanked(boolean thanked, Long id);
|
||||
int updateThanked(boolean thanked, Long id);
|
||||
|
||||
int updateRank(int rank, Long id);
|
||||
}
|
||||
|
||||
+6
-6
@@ -6,15 +6,15 @@ import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IQueuedRepository extends IRepository {
|
||||
Queued save(Queued queued);
|
||||
Queued save(Queued queued);
|
||||
|
||||
Optional<Queued> findByThreadId(String threadId);
|
||||
Optional<Queued> findByThreadId(String threadId);
|
||||
|
||||
List<Queued> findAll();
|
||||
List<Queued> findAll();
|
||||
|
||||
Optional<Queued> findById(Long id);
|
||||
Optional<Queued> findById(Long id);
|
||||
|
||||
int deleteByThreadId(String threadId);
|
||||
int deleteByThreadId(String threadId);
|
||||
|
||||
void deleteAll();
|
||||
void deleteAll();
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
package tn.mnlr.vripper.jpa.repositories;
|
||||
|
||||
public interface IRepository {
|
||||
}
|
||||
public interface IRepository {}
|
||||
|
||||
-117
@@ -1,117 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.EventRemoveEvent;
|
||||
import tn.mnlr.vripper.event.EventUpdateEvent;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class EventRepository implements IEventRepository, ApplicationEventPublisherAware {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final SettingsService settingsService;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
public EventRepository(JdbcTemplate jdbcTemplate, SettingsService settingsService) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.settingsService = settingsService;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"CALL NEXT VALUE FOR SEQ_EVENT",
|
||||
Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Event save(@NonNull Event event) {
|
||||
|
||||
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
|
||||
|
||||
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
|
||||
if (count > maxRecords) {
|
||||
List<Long> idList = jdbcTemplate.queryForList("SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
|
||||
idList.forEach(this::delete);
|
||||
}
|
||||
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
|
||||
id,
|
||||
event.getType().name(),
|
||||
event.getStatus().name(),
|
||||
event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
event.getMessage()
|
||||
);
|
||||
event.setId(id);
|
||||
applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, id));
|
||||
return event;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Event update(@NonNull Event event) {
|
||||
if (event.getId() == null) {
|
||||
log.warn("Cannot update entity with null id");
|
||||
return event;
|
||||
}
|
||||
|
||||
jdbcTemplate.update(
|
||||
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
|
||||
event.getType().name(),
|
||||
event.getStatus().name(),
|
||||
event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
event.getMessage(),
|
||||
event.getId()
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, event.getId()));
|
||||
return event;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Event> findById(Long id) {
|
||||
List<Event> events = jdbcTemplate.query(
|
||||
"SELECT * FROM EVENT WHERE ID = ?",
|
||||
new EventRowMapper(),
|
||||
id);
|
||||
if (events.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(events.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Event> findAll() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM EVENT",
|
||||
new EventRowMapper());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(Long id) {
|
||||
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
|
||||
applicationEventPublisher.publishEvent(new EventRemoveEvent(EventRepository.class, id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM EVENT");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Event;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public class EventRowMapper implements RowMapper<Event> {
|
||||
|
||||
@Override
|
||||
public Event mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Event event = new Event();
|
||||
event.setId(rs.getLong("ID"));
|
||||
event.setType(Event.Type.valueOf(rs.getString("TYPE")));
|
||||
event.setStatus(Event.Status.valueOf(rs.getString("STATUS")));
|
||||
event.setTime(LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
|
||||
event.setMessage(rs.getString("MESSAGE"));
|
||||
return event;
|
||||
}
|
||||
}
|
||||
+135
-125
@@ -1,146 +1,156 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.ImageUpdateEvent;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware {
|
||||
public class ImageRepository implements IImageRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public ImageRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
@Autowired
|
||||
public ImageRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_IMAGE", Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Image save(Image image) {
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
image.getCurrent(),
|
||||
image.getHost().getHost(),
|
||||
image.getIndex(),
|
||||
image.getPostId(),
|
||||
image.getStatus().name(),
|
||||
image.getTotal(),
|
||||
image.getUrl(),
|
||||
image.getPostIdRef());
|
||||
image.setId(id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return image;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAllByPostId(String postId) {
|
||||
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostId(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE WHERE POST_ID = ?", new ImageRowMapper(), postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countError() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'", Integer.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int stopByPostIdAndIsNotCompleted(String postId) {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostIdAndIsError(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Image> findById(Long id) {
|
||||
List<Image> images =
|
||||
jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.ID = ?", new ImageRowMapper(), id);
|
||||
if (images.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(images.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"CALL NEXT VALUE FOR SEQ_IMAGE",
|
||||
Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Image save(Image image) {
|
||||
long id = nextId();
|
||||
@Override
|
||||
public int updateStatus(Status status, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
image.getCurrent(),
|
||||
image.getHost().getHost(),
|
||||
image.getIndex(),
|
||||
image.getPostId(),
|
||||
image.getStatus().name(),
|
||||
image.getTotal(),
|
||||
image.getUrl(),
|
||||
image.getPostIdRef()
|
||||
);
|
||||
image.setId(id);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return image;
|
||||
}
|
||||
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?", status.name(), id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAllByPostId(String postId) {
|
||||
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
|
||||
}
|
||||
@Override
|
||||
public int updateCurrent(long current, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?", current, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostId(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE WHERE POST_ID = ?",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countError() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'",
|
||||
Integer.class
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int stopByPostIdAndIsNotCompleted(String postId) {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
|
||||
postId
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Image> findByPostIdAndIsError(String postId) {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
|
||||
new ImageRowMapper(),
|
||||
postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Image> findById(Long id) {
|
||||
List<Image> images = jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.ID = ?",
|
||||
new ImageRowMapper(),
|
||||
id);
|
||||
if (images.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(images.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateStatus(Status status, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?",
|
||||
status.name(), id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateCurrent(long current, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?",
|
||||
current, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateTotal(long total, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?",
|
||||
total, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
@Override
|
||||
public int updateTotal(long total, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?", total, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
}
|
||||
|
||||
class ImageRowMapper implements RowMapper<Image> {
|
||||
|
||||
@Override
|
||||
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Image image = new Image();
|
||||
image.setId(rs.getLong("ID"));
|
||||
String host = rs.getString("HOST");
|
||||
image.setHost(
|
||||
SpringContext.getBeansOfType(Host.class).values().stream()
|
||||
.filter(e -> e.getHost().equals(host))
|
||||
.findAny()
|
||||
.orElse(null));
|
||||
image.setUrl(rs.getString("URL"));
|
||||
image.setIndex(rs.getInt("INDEX"));
|
||||
image.setCurrent(rs.getLong("CURRENT"));
|
||||
image.setTotal(rs.getLong("TOTAL"));
|
||||
image.setStatus(Status.valueOf(rs.getString("STATUS")));
|
||||
image.setPostId(rs.getString("POST_ID"));
|
||||
image.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
return image;
|
||||
}
|
||||
}
|
||||
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.SpringContext;
|
||||
import tn.mnlr.vripper.host.Host;
|
||||
import tn.mnlr.vripper.jpa.domain.Image;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class ImageRowMapper implements RowMapper<Image> {
|
||||
|
||||
@Override
|
||||
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Image image = new Image();
|
||||
image.setId(rs.getLong("ID"));
|
||||
String host = rs.getString("HOST");
|
||||
image.setHost(SpringContext.getBeansOfType(Host.class).values().stream().filter(e -> e.getHost().equals(host)).findAny().orElse(null));
|
||||
image.setUrl(rs.getString("URL"));
|
||||
image.setIndex(rs.getInt("INDEX"));
|
||||
image.setCurrent(rs.getLong("CURRENT"));
|
||||
image.setTotal(rs.getLong("TOTAL"));
|
||||
image.setStatus(Status.valueOf(rs.getString("STATUS")));
|
||||
image.setPostId(rs.getString("POST_ID"));
|
||||
image.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
return image;
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.LogEvent;
|
||||
import tn.mnlr.vripper.jpa.repositories.ILogEventRepository;
|
||||
import tn.mnlr.vripper.services.SettingsService;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class LogEventRepository implements ILogEventRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final SettingsService settingsService;
|
||||
private final EventBus eventBus;
|
||||
|
||||
public LogEventRepository(
|
||||
JdbcTemplate jdbcTemplate, SettingsService settingsService, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.settingsService = settingsService;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_EVENT", Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized LogEvent save(@NonNull LogEvent logEvent) {
|
||||
|
||||
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
|
||||
|
||||
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
|
||||
if (count > maxRecords) {
|
||||
List<Long> idList =
|
||||
jdbcTemplate.queryForList(
|
||||
"SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
|
||||
idList.forEach(this::delete);
|
||||
}
|
||||
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
|
||||
id,
|
||||
logEvent.getType().name(),
|
||||
logEvent.getStatus().name(),
|
||||
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
logEvent.getMessage());
|
||||
logEvent.setId(id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, id));
|
||||
return logEvent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public LogEvent update(@NonNull LogEvent logEvent) {
|
||||
if (logEvent.getId() == null) {
|
||||
log.warn("Cannot update entity with null id");
|
||||
return logEvent;
|
||||
}
|
||||
|
||||
jdbcTemplate.update(
|
||||
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
|
||||
logEvent.getType().name(),
|
||||
logEvent.getStatus().name(),
|
||||
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
logEvent.getMessage(),
|
||||
logEvent.getId());
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, logEvent.getId()));
|
||||
return logEvent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<LogEvent> findById(Long id) {
|
||||
List<LogEvent> logEvents =
|
||||
jdbcTemplate.query("SELECT * FROM EVENT WHERE ID = ?", new LogEventRowMapper(), id);
|
||||
if (logEvents.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(logEvents.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LogEvent> findAll() {
|
||||
return jdbcTemplate.query("SELECT * FROM EVENT", new LogEventRowMapper());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(Long id) {
|
||||
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_REMOVE, id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM EVENT");
|
||||
}
|
||||
}
|
||||
|
||||
class LogEventRowMapper implements RowMapper<LogEvent> {
|
||||
|
||||
@Override
|
||||
public LogEvent mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
LogEvent logEvent = new LogEvent();
|
||||
logEvent.setId(rs.getLong("ID"));
|
||||
logEvent.setType(LogEvent.Type.valueOf(rs.getString("TYPE")));
|
||||
logEvent.setStatus(LogEvent.Status.valueOf(rs.getString("STATUS")));
|
||||
logEvent.setTime(
|
||||
LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
|
||||
logEvent.setMessage(rs.getString("MESSAGE"));
|
||||
return logEvent;
|
||||
}
|
||||
}
|
||||
+56
-45
@@ -1,65 +1,76 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.MetadataUpdateEvent;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware {
|
||||
public class MetadataRepository implements IMetadataRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public MetadataRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
@Autowired
|
||||
public MetadataRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Metadata save(Metadata metadata) {
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
|
||||
metadata.getPostIdRef(),
|
||||
metadata.getPostId(),
|
||||
metadata.getPostedBy(),
|
||||
String.join("%sep%", metadata.getResolvedNames()));
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.METADATA_UPDATE, metadata.getPostIdRef()));
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Metadata> findByPostId(String postId) {
|
||||
List<Metadata> metadata =
|
||||
jdbcTemplate.query(
|
||||
"SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
|
||||
new MetadataRowMapper(),
|
||||
postId);
|
||||
if (metadata.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(metadata.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Metadata save(Metadata metadata) {
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
|
||||
metadata.getPostIdRef(),
|
||||
metadata.getPostId(),
|
||||
metadata.getPostedBy(),
|
||||
String.join("%sep%", metadata.getResolvedNames())
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, metadata.getPostIdRef()));
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Metadata> findByPostId(String postId) {
|
||||
List<Metadata> metadata = jdbcTemplate.query(
|
||||
"SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
|
||||
new MetadataRowMapper(),
|
||||
postId
|
||||
);
|
||||
if (metadata.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(metadata.get(0));
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public int deleteByPostId(String postId) {
|
||||
return jdbcTemplate.update(
|
||||
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?", postId);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByPostId(String postId) {
|
||||
return jdbcTemplate.update(
|
||||
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
|
||||
postId
|
||||
);
|
||||
}
|
||||
class MetadataRowMapper implements RowMapper<Metadata> {
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
@Override
|
||||
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
metadata.setPostId(rs.getString("POST_ID"));
|
||||
metadata.setPostedBy(rs.getString("POSTED_BY"));
|
||||
String resolvedNames = rs.getString("RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
public class MetadataRowMapper implements RowMapper<Metadata> {
|
||||
|
||||
@Override
|
||||
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
|
||||
metadata.setPostId(rs.getString("POST_ID"));
|
||||
metadata.setPostedBy(rs.getString("POSTED_BY"));
|
||||
String resolvedNames = rs.getString("RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
+203
-161
@@ -1,188 +1,230 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.PostRemoveEvent;
|
||||
import tn.mnlr.vripper.event.PostUpdateEvent;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
public class PostRepository implements IPostRepository, ApplicationEventPublisherAware {
|
||||
public class PostRepository implements IPostRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public PostRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
@Autowired
|
||||
public PostRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_POST", Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Post save(Post post) {
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON, RANK) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
post.getDone(),
|
||||
post.getForum(),
|
||||
String.join(";", post.getHosts()),
|
||||
post.getDownloadDirectory(),
|
||||
post.getPostId(),
|
||||
String.join(";", post.getPreviews()),
|
||||
post.getSecurityToken(),
|
||||
post.getStatus().name(),
|
||||
post.isThanked(),
|
||||
post.getThreadId(),
|
||||
post.getThreadTitle(),
|
||||
post.getTitle(),
|
||||
post.getTotal(),
|
||||
post.getUrl(),
|
||||
Timestamp.valueOf(post.getAddedOn()),
|
||||
post.getRank());
|
||||
post.setId(id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return post;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Post> findByPostId(String postId) {
|
||||
List<Post> posts =
|
||||
jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
|
||||
new PostRowMapper(),
|
||||
postId);
|
||||
if (posts.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(posts.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"CALL NEXT VALUE FOR SEQ_POST",
|
||||
Long.class);
|
||||
@Override
|
||||
public List<String> findCompleted() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total",
|
||||
((rs, rowNum) -> rs.getString("POST_ID")));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Post> findById(Long id) {
|
||||
List<Post> posts =
|
||||
jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
|
||||
new PostRowMapper(),
|
||||
id);
|
||||
if (posts.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(posts.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Post save(Post post) {
|
||||
long id = nextId();
|
||||
@Override
|
||||
public List<Post> findAll() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
|
||||
new PostRowMapper());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean existByPostId(String postId) {
|
||||
Integer count =
|
||||
jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?", Integer.class, postId);
|
||||
if (count == null) {
|
||||
return false;
|
||||
} else {
|
||||
return count > 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDownloadingToStopped() {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByPostId(String postId) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("DELETE FROM POST AS post WHERE post.POST_ID = ?", postId);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_REMOVE, postId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateStatus(Status status, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
post.getDone(),
|
||||
post.getForum(),
|
||||
String.join(";", post.getHosts()),
|
||||
post.getDownloadDirectory(),
|
||||
post.getPostId(),
|
||||
String.join(";", post.getPreviews()),
|
||||
post.getSecurityToken(),
|
||||
post.getStatus().name(),
|
||||
post.isThanked(),
|
||||
post.getThreadId(),
|
||||
post.getThreadTitle(),
|
||||
post.getTitle(),
|
||||
post.getTotal(),
|
||||
post.getUrl()
|
||||
);
|
||||
post.setId(id);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return post;
|
||||
}
|
||||
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?", status.name(), id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Post> findByPostId(String postId) {
|
||||
List<Post> posts = jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
|
||||
new PostRowMapper(),
|
||||
postId);
|
||||
if (posts.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(posts.get(0));
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public int updateDone(int done, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?", done, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> findCompleted() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total",
|
||||
((rs, rowNum) -> rs.getString("POST_ID"))
|
||||
);
|
||||
}
|
||||
@Override
|
||||
public int updateFolderName(String postFolderName, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
|
||||
postFolderName,
|
||||
id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Post> findById(Long id) {
|
||||
List<Post> posts = jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
|
||||
new PostRowMapper(),
|
||||
id);
|
||||
if (posts.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(posts.get(0));
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public int updateTitle(String title, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?", title, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Post> findAll() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
|
||||
new PostRowMapper()
|
||||
);
|
||||
}
|
||||
@Override
|
||||
public int updateThanked(boolean thanked, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?", thanked, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean existByPostId(String postId) {
|
||||
Integer count = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?",
|
||||
Integer.class,
|
||||
postId);
|
||||
if (count == null) {
|
||||
return false;
|
||||
} else {
|
||||
return count > 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDownloadingToStopped() {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'"
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByPostId(String postId) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"DELETE FROM POST AS post WHERE post.POST_ID = ?",
|
||||
postId
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostRemoveEvent(PostRepository.class, postId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateStatus(Status status, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?",
|
||||
status.name(), id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateDone(int done, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?",
|
||||
done, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateFolderName(String postFolderName, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
|
||||
postFolderName, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateTitle(String title, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?",
|
||||
title, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateThanked(boolean thanked, Long id) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?",
|
||||
thanked, id
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
@Override
|
||||
public int updateRank(int rank, Long id) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("UPDATE POST AS post SET post.RANK = ? WHERE post.ID = ?", rank, id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
|
||||
return mutationCount;
|
||||
}
|
||||
}
|
||||
|
||||
class PostRowMapper implements RowMapper<Post> {
|
||||
|
||||
private static final String DELIMITER = ";";
|
||||
|
||||
@Override
|
||||
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
|
||||
Post post = new Post();
|
||||
post.setId(rs.getLong("post.ID"));
|
||||
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
|
||||
post.setPostId(rs.getString("post.POST_ID"));
|
||||
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
|
||||
post.setThreadId(rs.getString("post.THREAD_ID"));
|
||||
post.setTitle(rs.getString("post.TITLE"));
|
||||
post.setUrl(rs.getString("post.URL"));
|
||||
post.setDone(rs.getInt("post.DONE"));
|
||||
post.setTotal(rs.getInt("post.TOTAL"));
|
||||
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
|
||||
post.setForum(rs.getString("post.FORUM"));
|
||||
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
|
||||
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
|
||||
post.setThanked(rs.getBoolean("post.THANKED"));
|
||||
String previews;
|
||||
if ((previews = rs.getString("post.PREVIEWS")) != null) {
|
||||
post.setPreviews(Set.of(previews.split(DELIMITER)));
|
||||
}
|
||||
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
|
||||
post.setRank(rs.getInt("post.RANK"));
|
||||
|
||||
Long metadataId = rs.getLong("metadata.POST_ID_REF");
|
||||
if (!rs.wasNull()) {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(metadataId);
|
||||
metadata.setPostId(rs.getString("metadata.POST_ID"));
|
||||
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
|
||||
post.setMetadata(metadata);
|
||||
}
|
||||
|
||||
return post;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.jpa.domain.enums.Status;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class PostRowMapper implements RowMapper<Post> {
|
||||
|
||||
private static final String DELIMITER = ";";
|
||||
|
||||
@Override
|
||||
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
|
||||
Post post = new Post();
|
||||
post.setId(rs.getLong("post.ID"));
|
||||
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
|
||||
post.setPostId(rs.getString("post.POST_ID"));
|
||||
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
|
||||
post.setThreadId(rs.getString("post.THREAD_ID"));
|
||||
post.setTitle(rs.getString("post.TITLE"));
|
||||
post.setUrl(rs.getString("post.URL"));
|
||||
post.setDone(rs.getInt("post.DONE"));
|
||||
post.setTotal(rs.getInt("post.TOTAL"));
|
||||
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
|
||||
post.setForum(rs.getString("post.FORUM"));
|
||||
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
|
||||
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
|
||||
post.setThanked(rs.getBoolean("post.THANKED"));
|
||||
String previews;
|
||||
if ((previews = rs.getString("post.PREVIEWS")) != null) {
|
||||
post.setPreviews(Set.of(previews.split(DELIMITER)));
|
||||
}
|
||||
|
||||
Long metadataId = rs.getLong("metadata.POST_ID_REF");
|
||||
if (!rs.wasNull()) {
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setPostIdRef(metadataId);
|
||||
metadata.setPostId(rs.getString("metadata.POST_ID"));
|
||||
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
|
||||
if (resolvedNames != null && !resolvedNames.isBlank()) {
|
||||
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
|
||||
}
|
||||
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
|
||||
post.setMetadata(metadata);
|
||||
}
|
||||
|
||||
return post;
|
||||
}
|
||||
}
|
||||
+84
-82
@@ -1,105 +1,107 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.ApplicationEventPublisherAware;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.event.QueuedRemoveEvent;
|
||||
import tn.mnlr.vripper.event.QueuedUpdateEvent;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware {
|
||||
public class QueuedRepository implements IQueuedRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private ApplicationEventPublisher applicationEventPublisher;
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
public QueuedRepository(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
@Autowired
|
||||
public QueuedRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_QUEUED", Long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Queued save(Queued queued) {
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
|
||||
id,
|
||||
queued.getTotal(),
|
||||
queued.getLink(),
|
||||
queued.isLoading(),
|
||||
queued.getPostId(),
|
||||
queued.getThreadId());
|
||||
queued.setId(id);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_UPDATE, id));
|
||||
return queued;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Queued> findByThreadId(String threadId) {
|
||||
List<Queued> queuedList =
|
||||
jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
|
||||
new QueuedRowMapper(),
|
||||
threadId);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(queuedList.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized Long nextId() {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"CALL NEXT VALUE FOR SEQ_QUEUED",
|
||||
Long.class);
|
||||
}
|
||||
@Override
|
||||
public List<Queued> findAll() {
|
||||
return jdbcTemplate.query("SELECT * FROM QUEUED", new QueuedRowMapper());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Queued save(Queued queued) {
|
||||
long id = nextId();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
|
||||
id,
|
||||
queued.getTotal(),
|
||||
queued.getLink(),
|
||||
queued.isLoading(),
|
||||
queued.getPostId(),
|
||||
queued.getThreadId()
|
||||
);
|
||||
queued.setId(id);
|
||||
applicationEventPublisher.publishEvent(new QueuedUpdateEvent(QueuedRepository.class, id));
|
||||
return queued;
|
||||
@Override
|
||||
public Optional<Queued> findById(Long id) {
|
||||
List<Queued> queuedList =
|
||||
jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?", new QueuedRowMapper(), id);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(queuedList.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Queued> findByThreadId(String threadId) {
|
||||
List<Queued> queuedList = jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
|
||||
new Object[]{threadId},
|
||||
new QueuedRowMapper()
|
||||
);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(queuedList.get(0));
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public int deleteByThreadId(String threadId) {
|
||||
int mutationCount =
|
||||
jdbcTemplate.update("DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?", threadId);
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_REMOVE, threadId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Queued> findAll() {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED",
|
||||
new QueuedRowMapper()
|
||||
);
|
||||
}
|
||||
@Override
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM QUEUED");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Queued> findById(Long id) {
|
||||
List<Queued> queuedList = jdbcTemplate.query(
|
||||
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?",
|
||||
new Object[]{id},
|
||||
new QueuedRowMapper()
|
||||
);
|
||||
if (queuedList.isEmpty()) {
|
||||
return Optional.empty();
|
||||
} else {
|
||||
return Optional.of(queuedList.get(0));
|
||||
}
|
||||
}
|
||||
class QueuedRowMapper implements RowMapper<Queued> {
|
||||
|
||||
@Override
|
||||
public int deleteByThreadId(String threadId) {
|
||||
int mutationCount = jdbcTemplate.update(
|
||||
"DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?",
|
||||
threadId
|
||||
);
|
||||
applicationEventPublisher.publishEvent(new QueuedRemoveEvent(QueuedRepository.class, threadId));
|
||||
return mutationCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM QUEUED");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
@Override
|
||||
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Queued queued = new Queued();
|
||||
queued.setId(rs.getLong("ID"));
|
||||
queued.setLink(rs.getString("LINK"));
|
||||
queued.setThreadId(rs.getString("THREAD_ID"));
|
||||
queued.setPostId(rs.getString("POST_ID"));
|
||||
queued.setTotal(rs.getInt("TOTAL"));
|
||||
queued.setLoading(rs.getBoolean("LOADING"));
|
||||
return queued;
|
||||
}
|
||||
}
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package tn.mnlr.vripper.jpa.repositories.impl;
|
||||
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
public class QueuedRowMapper implements RowMapper<Queued> {
|
||||
|
||||
@Override
|
||||
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
Queued queued = new Queued();
|
||||
queued.setId(rs.getLong("ID"));
|
||||
queued.setLink(rs.getString("LINK"));
|
||||
queued.setThreadId(rs.getString("THREAD_ID"));
|
||||
queued.setPostId(rs.getString("POST_ID"));
|
||||
queued.setTotal(rs.getInt("TOTAL"));
|
||||
queued.setLoading(rs.getBoolean("LOADING"));
|
||||
return queued;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public interface DataEventListener<T> {
|
||||
Flux<T> getDataFlux();
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import reactor.core.publisher.Sinks.EmitFailureHandler;
|
||||
|
||||
public class EmitHandler {
|
||||
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.EventRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EventRemoveEventListener implements ApplicationListener<EventRemoveEvent>, DataEventListener<EventRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<EventRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(EventRemoveEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<EventRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.EventUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EventUpdateEventListener implements ApplicationListener<EventUpdateEvent>, DataEventListener<EventUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<EventUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(@NonNull EventUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<EventUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.ImageUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class ImageUpdateEventListener implements ApplicationListener<ImageUpdateEvent>, DataEventListener<ImageUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<ImageUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(ImageUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ImageUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.MetadataUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class MetadataUpdateEventListener implements ApplicationListener<MetadataUpdateEvent>, DataEventListener<MetadataUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<MetadataUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(MetadataUpdateEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<MetadataUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.PostRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class PostRemoveEventListener implements ApplicationListener<PostRemoveEvent>, DataEventListener<PostRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<PostRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(PostRemoveEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<PostRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.PostUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class PostUpdateEventListener implements ApplicationListener<PostUpdateEvent>, DataEventListener<PostUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<PostUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(PostUpdateEvent event) {
|
||||
sink.emitNext(event, EmitHandler.RETRY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<PostUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.QueuedRemoveEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueuedRemoveEventListener implements ApplicationListener<QueuedRemoveEvent>, DataEventListener<QueuedRemoveEvent> {
|
||||
|
||||
private final Sinks.Many<QueuedRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(QueuedRemoveEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<QueuedRemoveEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package tn.mnlr.vripper.listener;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.event.QueuedUpdateEvent;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueuedUpdateEventListener implements ApplicationListener<QueuedUpdateEvent>, DataEventListener<QueuedUpdateEvent> {
|
||||
|
||||
private final Sinks.Many<QueuedUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(QueuedUpdateEvent event) {
|
||||
sink.emitNext(event, (signalType, emitResult) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<QueuedUpdateEvent> getDataFlux() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,10 @@ import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.Disposable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import tn.mnlr.vripper.download.DownloadJob;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.net.URI;
|
||||
@@ -31,104 +33,122 @@ import java.util.concurrent.TimeUnit;
|
||||
@Slf4j
|
||||
public class ConnectionService {
|
||||
|
||||
private final Disposable disposable;
|
||||
private PoolingHttpClientConnectionManager pcm;
|
||||
private RequestConfig rc;
|
||||
private final Disposable disposable;
|
||||
private PoolingHttpClientConnectionManager pcm;
|
||||
private RequestConfig rc;
|
||||
|
||||
@Getter
|
||||
private RetryPolicy<Object> retryPolicy;
|
||||
@Getter private RetryPolicy<Object> retryPolicy;
|
||||
|
||||
private int connectionTimeout;
|
||||
private int maxAttempts;
|
||||
private int connectionTimeout;
|
||||
private int maxAttempts;
|
||||
|
||||
public ConnectionService(SettingsService settingsService) {
|
||||
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
|
||||
maxAttempts = settingsService.getSettings().getMaxAttempts();
|
||||
Flux<SettingsService.Settings> settingsFlux = settingsService.getSettingsFlux();
|
||||
disposable = settingsFlux.subscribe(settings -> {
|
||||
if (connectionTimeout != settings.getConnectionTimeout()) {
|
||||
connectionTimeout = settings.getConnectionTimeout();
|
||||
buildRequestConfig();
|
||||
}
|
||||
if (maxAttempts != settings.getMaxAttempts()) {
|
||||
maxAttempts = settings.getMaxAttempts();
|
||||
buildRetryPolicy();
|
||||
}
|
||||
});
|
||||
public ConnectionService(EventBus eventBus, SettingsService settingsService) {
|
||||
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
|
||||
maxAttempts = settingsService.getSettings().getMaxAttempts();
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.map(e -> ((Settings) e.getData()))
|
||||
.subscribe(
|
||||
settings -> {
|
||||
if (connectionTimeout != settings.getConnectionTimeout()) {
|
||||
connectionTimeout = settings.getConnectionTimeout();
|
||||
buildRequestConfig();
|
||||
}
|
||||
if (maxAttempts != settings.getMaxAttempts()) {
|
||||
maxAttempts = settings.getMaxAttempts();
|
||||
buildRetryPolicy();
|
||||
}
|
||||
});
|
||||
|
||||
buildRequestConfig();
|
||||
buildRetryPolicy();
|
||||
buildConnectionPool();
|
||||
}
|
||||
buildRequestConfig();
|
||||
buildRetryPolicy();
|
||||
buildConnectionPool();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
disposable.dispose();
|
||||
}
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
disposable.dispose();
|
||||
}
|
||||
|
||||
private void buildRetryPolicy() {
|
||||
retryPolicy = new RetryPolicy<>()
|
||||
.withDelay(2, 5, ChronoUnit.SECONDS)
|
||||
.withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt(e -> log.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
|
||||
}
|
||||
private void buildRetryPolicy() {
|
||||
retryPolicy =
|
||||
new RetryPolicy<>()
|
||||
.withDelay(2, 5, ChronoUnit.SECONDS)
|
||||
.withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt(
|
||||
e ->
|
||||
log.warn(
|
||||
String.format("#%d tries failed", e.getAttemptCount()),
|
||||
e.getLastFailure()));
|
||||
}
|
||||
|
||||
private void buildConnectionPool() {
|
||||
pcm = new PoolingHttpClientConnectionManager();
|
||||
pcm.setMaxTotal(Integer.MAX_VALUE);
|
||||
pcm.setDefaultMaxPerRoute(Integer.MAX_VALUE);
|
||||
}
|
||||
private void buildConnectionPool() {
|
||||
pcm = new PoolingHttpClientConnectionManager();
|
||||
pcm.setMaxTotal(Integer.MAX_VALUE);
|
||||
pcm.setDefaultMaxPerRoute(Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
private void buildRequestConfig() {
|
||||
rc = RequestConfig.custom()
|
||||
.setConnectionRequestTimeout(connectionTimeout * 1000)
|
||||
.setConnectTimeout(connectionTimeout * 1000)
|
||||
.setSocketTimeout(connectionTimeout * 1000)
|
||||
.setCookieSpec(CookieSpecs.STANDARD)
|
||||
.build();
|
||||
}
|
||||
private void buildRequestConfig() {
|
||||
rc =
|
||||
RequestConfig.custom()
|
||||
.setConnectionRequestTimeout(connectionTimeout * 1000)
|
||||
.setConnectTimeout(connectionTimeout * 1000)
|
||||
.setSocketTimeout(connectionTimeout * 1000)
|
||||
.setCookieSpec(CookieSpecs.STANDARD)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 5_000)
|
||||
private void idleConnectionMonitoring() {
|
||||
pcm.closeExpiredConnections();
|
||||
pcm.closeIdleConnections(30, TimeUnit.SECONDS);
|
||||
}
|
||||
@Scheduled(fixedDelay = 5_000)
|
||||
private void idleConnectionMonitoring() {
|
||||
pcm.closeExpiredConnections();
|
||||
pcm.closeIdleConnections(30, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
public HttpClientBuilder getClient() {
|
||||
return HttpClients.custom()
|
||||
.setConnectionManager(pcm)
|
||||
.setRedirectStrategy(new LaxRedirectStrategy())
|
||||
.disableAutomaticRetries()
|
||||
.setDefaultRequestConfig(rc);
|
||||
}
|
||||
public HttpClientBuilder getClient() {
|
||||
return HttpClients.custom()
|
||||
.setConnectionManager(pcm)
|
||||
.setRedirectStrategy(new LaxRedirectStrategy())
|
||||
.disableAutomaticRetries()
|
||||
.setDefaultRequestConfig(rc);
|
||||
}
|
||||
|
||||
public HttpGet buildHttpGet(String url, final HttpClientContext context) {
|
||||
HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
|
||||
httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
|
||||
addToContext(context, httpGet);
|
||||
return httpGet;
|
||||
}
|
||||
public HttpGet buildHttpGet(String url, final HttpClientContext context) {
|
||||
HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
|
||||
httpGet.addHeader(
|
||||
"User-Agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
|
||||
addToContext(context, httpGet);
|
||||
return httpGet;
|
||||
}
|
||||
|
||||
public HttpPost buildHttpPost(String url, final HttpClientContext context) {
|
||||
HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
|
||||
httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
|
||||
addToContext(context, httpPost);
|
||||
return httpPost;
|
||||
}
|
||||
public HttpPost buildHttpPost(String url, final HttpClientContext context) {
|
||||
HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
|
||||
httpPost.addHeader(
|
||||
"User-Agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
|
||||
addToContext(context, httpPost);
|
||||
return httpPost;
|
||||
}
|
||||
|
||||
public HttpGet buildHttpGet(URI uri, final HttpClientContext context) {
|
||||
HttpGet httpGet = new HttpGet(uri);
|
||||
httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
|
||||
addToContext(context, httpGet);
|
||||
return httpGet;
|
||||
}
|
||||
public HttpGet buildHttpGet(URI uri, final HttpClientContext context) {
|
||||
HttpGet httpGet = new HttpGet(uri);
|
||||
httpGet.addHeader(
|
||||
"User-Agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
|
||||
addToContext(context, httpGet);
|
||||
return httpGet;
|
||||
}
|
||||
|
||||
public void addToContext(HttpClientContext context, AbstractExecutionAwareRequest request) {
|
||||
if (context != null) {
|
||||
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
|
||||
if (requests != null) {
|
||||
requests.add(request);
|
||||
}
|
||||
}
|
||||
public void addToContext(HttpClientContext context, AbstractExecutionAwareRequest request) {
|
||||
if (context != null) {
|
||||
List<AbstractExecutionAwareRequest> requests =
|
||||
(List<AbstractExecutionAwareRequest>)
|
||||
context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
|
||||
if (requests != null) {
|
||||
requests.add(request);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,217 +11,239 @@ import tn.mnlr.vripper.jpa.repositories.IImageRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.impl.EventRepository;
|
||||
import tn.mnlr.vripper.jpa.repositories.impl.LogEventRepository;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Transactional
|
||||
@Slf4j
|
||||
public class DataService {
|
||||
|
||||
private final IPostRepository postRepository;
|
||||
private final IImageRepository imageRepository;
|
||||
private final IQueuedRepository queuedRepository;
|
||||
private final IMetadataRepository metadataRepository;
|
||||
private final SettingsService settingsService;
|
||||
private final EventRepository eventRepository;
|
||||
private final IPostRepository postRepository;
|
||||
private final IImageRepository imageRepository;
|
||||
private final IQueuedRepository queuedRepository;
|
||||
private final IMetadataRepository metadataRepository;
|
||||
private final SettingsService settingsService;
|
||||
private final LogEventRepository eventRepository;
|
||||
|
||||
@Autowired
|
||||
public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, SettingsService settingsService, EventRepository eventRepository) {
|
||||
this.postRepository = postRepository;
|
||||
this.imageRepository = imageRepository;
|
||||
this.queuedRepository = queuedRepository;
|
||||
this.metadataRepository = metadataRepository;
|
||||
this.settingsService = settingsService;
|
||||
this.eventRepository = eventRepository;
|
||||
}
|
||||
@Autowired
|
||||
public DataService(
|
||||
IPostRepository postRepository,
|
||||
IImageRepository imageRepository,
|
||||
IQueuedRepository queuedRepository,
|
||||
IMetadataRepository metadataRepository,
|
||||
SettingsService settingsService,
|
||||
LogEventRepository eventRepository) {
|
||||
this.postRepository = postRepository;
|
||||
this.imageRepository = imageRepository;
|
||||
this.queuedRepository = queuedRepository;
|
||||
this.metadataRepository = metadataRepository;
|
||||
this.settingsService = settingsService;
|
||||
this.eventRepository = eventRepository;
|
||||
}
|
||||
|
||||
private void save(Post post) {
|
||||
postRepository.save(post);
|
||||
}
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
sortPostsByRank();
|
||||
}
|
||||
|
||||
private void save(Queued queued) {
|
||||
queuedRepository.save(queued);
|
||||
}
|
||||
private void save(Post post) {
|
||||
postRepository.save(post);
|
||||
}
|
||||
|
||||
private void save(Image image) {
|
||||
imageRepository.save(image);
|
||||
}
|
||||
private void save(Queued queued) {
|
||||
queuedRepository.save(queued);
|
||||
}
|
||||
|
||||
public boolean exists(String postId) {
|
||||
return postRepository.existByPostId(postId);
|
||||
}
|
||||
private void save(Image image) {
|
||||
imageRepository.save(image);
|
||||
}
|
||||
|
||||
public void newPost(Post post, Collection<Image> images) {
|
||||
save(post);
|
||||
images.forEach(image -> {
|
||||
image.setPostIdRef(post.getId());
|
||||
save(image);
|
||||
public boolean exists(String postId) {
|
||||
return postRepository.existByPostId(postId);
|
||||
}
|
||||
|
||||
public void newPost(Post post, Collection<Image> images) {
|
||||
save(post);
|
||||
images.forEach(
|
||||
image -> {
|
||||
image.setPostIdRef(post.getId());
|
||||
save(image);
|
||||
});
|
||||
}
|
||||
sortPostsByRank();
|
||||
}
|
||||
|
||||
public void setDownloadingToStopped() {
|
||||
postRepository.setDownloadingToStopped();
|
||||
public synchronized void afterJobFinish(Image image, Post post) {
|
||||
if (image.getStatus().equals(Status.COMPLETE)) {
|
||||
post.setDone(post.getDone() + 1);
|
||||
updatePostDone(post.getDone(), post.getId());
|
||||
} else if (image.getStatus().equals(Status.ERROR)) {
|
||||
post.setStatus(Status.PARTIAL);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void afterJobFinish(Image image, Post post) {
|
||||
if (image.getStatus().equals(Status.COMPLETE)) {
|
||||
post.setDone(post.getDone() + 1);
|
||||
updatePostDone(post.getDone(), post.getId());
|
||||
} else if (image.getStatus().equals(Status.ERROR)) {
|
||||
post.setStatus(Status.PARTIAL);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
private void updatePostDone(int done, Long id) {
|
||||
postRepository.updateDone(done, id);
|
||||
}
|
||||
|
||||
public void updatePostStatus(Status status, Long id) {
|
||||
postRepository.updateStatus(status, id);
|
||||
}
|
||||
|
||||
public void updatePostThanks(boolean thanked, Long id) {
|
||||
postRepository.updateThanked(thanked, id);
|
||||
}
|
||||
|
||||
public void updatePostTitle(String title, Long id) {
|
||||
postRepository.updateTitle(title, id);
|
||||
}
|
||||
|
||||
public void updatePostDownloadDirectory(String postFolderName, Long id) {
|
||||
postRepository.updateFolderName(postFolderName, id);
|
||||
}
|
||||
|
||||
public void updatePostRank(int rank, Long id) {
|
||||
postRepository.updateRank(rank, id);
|
||||
}
|
||||
|
||||
public void finishPost(@NonNull Post post) {
|
||||
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
|
||||
post.setStatus(Status.ERROR);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
} else {
|
||||
if (post.getDone() < post.getTotal()) {
|
||||
post.setStatus(Status.STOPPED);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
} else {
|
||||
post.setStatus(Status.COMPLETE);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
if (settingsService.getSettings().getClearCompleted()) {
|
||||
remove(List.of(post.getPostId()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void updatePostDone(int done, Long id) {
|
||||
postRepository.updateDone(done, id);
|
||||
private void remove(@NonNull final List<String> postIds) {
|
||||
for (String postId : postIds) {
|
||||
imageRepository.deleteAllByPostId(postId);
|
||||
metadataRepository.deleteByPostId(postId);
|
||||
postRepository.deleteByPostId(postId);
|
||||
}
|
||||
sortPostsByRank();
|
||||
}
|
||||
|
||||
public void finishPost(@NonNull Post post) {
|
||||
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
|
||||
post.setStatus(Status.ERROR);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
} else {
|
||||
if (post.getDone() < post.getTotal()) {
|
||||
post.setStatus(Status.STOPPED);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
} else {
|
||||
post.setStatus(Status.COMPLETE);
|
||||
updatePostStatus(post.getStatus(), post.getId());
|
||||
if (settingsService.getSettings().getClearCompleted()) {
|
||||
remove(post.getPostId());
|
||||
}
|
||||
}
|
||||
}
|
||||
public void newQueueLink(@NonNull final Queued queued) {
|
||||
save(queued);
|
||||
}
|
||||
|
||||
public void removeQueueLink(@NonNull final String threadId) {
|
||||
queuedRepository.deleteByThreadId(threadId);
|
||||
}
|
||||
|
||||
public List<String> clearCompleted() {
|
||||
List<String> completed = postRepository.findCompleted();
|
||||
remove(completed);
|
||||
return completed;
|
||||
}
|
||||
|
||||
public void removeAll(final List<String> postIds) {
|
||||
|
||||
remove(
|
||||
Objects.requireNonNullElse(
|
||||
postIds,
|
||||
postRepository.findAll().stream().map(Post::getPostId).collect(Collectors.toList())));
|
||||
}
|
||||
|
||||
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
|
||||
return imageRepository.findByPostIdAndIsNotCompleted(postId);
|
||||
}
|
||||
|
||||
public long countErrorImages() {
|
||||
return imageRepository.countError();
|
||||
}
|
||||
|
||||
public List<Image> findImagesByPostId(String postId) {
|
||||
return imageRepository.findByPostId(postId);
|
||||
}
|
||||
|
||||
public List<Post> findAllPosts() {
|
||||
return postRepository.findAll();
|
||||
}
|
||||
|
||||
public Optional<Post> findPostByPostId(String postId) {
|
||||
return postRepository.findByPostId(postId);
|
||||
}
|
||||
|
||||
public void stopImagesByPostIdAndIsNotCompleted(String postId) {
|
||||
imageRepository.stopByPostIdAndIsNotCompleted(postId);
|
||||
}
|
||||
|
||||
public Optional<Queued> findQueuedByThreadId(String threadId) {
|
||||
return queuedRepository.findByThreadId(threadId);
|
||||
}
|
||||
|
||||
public List<Queued> findAllQueued() {
|
||||
return queuedRepository.findAll();
|
||||
}
|
||||
|
||||
public Optional<Post> findById(Long aLong) {
|
||||
return postRepository.findById(aLong);
|
||||
}
|
||||
|
||||
public Optional<Image> findImageById(Long aLong) {
|
||||
return imageRepository.findById(aLong);
|
||||
}
|
||||
|
||||
public Optional<Queued> findQueuedById(Long aLong) {
|
||||
return queuedRepository.findById(aLong);
|
||||
}
|
||||
|
||||
public synchronized void setMetadata(Post post, Metadata metadata) {
|
||||
if (metadataRepository.findByPostId(post.getPostId()).isEmpty()) {
|
||||
metadata.setPostIdRef(post.getId());
|
||||
metadataRepository.save(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private void remove(@NonNull final String postId) {
|
||||
imageRepository.deleteAllByPostId(postId);
|
||||
metadataRepository.deleteByPostId(postId);
|
||||
postRepository.deleteByPostId(postId);
|
||||
}
|
||||
|
||||
public void newQueueLink(@NonNull final Queued queued) {
|
||||
save(queued);
|
||||
}
|
||||
|
||||
public void removeQueueLink(@NonNull final String threadId) {
|
||||
queuedRepository.deleteByThreadId(threadId);
|
||||
}
|
||||
|
||||
public List<String> clearCompleted() {
|
||||
List<String> completed = postRepository.findCompleted();
|
||||
completed.forEach(this::remove);
|
||||
return completed;
|
||||
}
|
||||
|
||||
public void removeAll(final List<String> postIds) {
|
||||
if (postIds != null && !postIds.isEmpty()) {
|
||||
for (String postId : postIds) {
|
||||
remove(postId);
|
||||
}
|
||||
} else {
|
||||
postRepository.findAll().forEach(p -> remove(p.getPostId()));
|
||||
}
|
||||
}
|
||||
|
||||
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
|
||||
return imageRepository.findByPostIdAndIsNotCompleted(postId);
|
||||
|
||||
}
|
||||
|
||||
public long countErrorImages() {
|
||||
return imageRepository.countError();
|
||||
}
|
||||
|
||||
public List<Image> findImagesByPostId(String postId) {
|
||||
return imageRepository.findByPostId(postId);
|
||||
}
|
||||
|
||||
public List<Post> findAllPosts() {
|
||||
return postRepository.findAll();
|
||||
}
|
||||
|
||||
public Optional<Post> findPostByPostId(String postId) {
|
||||
return postRepository.findByPostId(postId);
|
||||
}
|
||||
|
||||
public void stopImagesByPostIdAndIsNotCompleted(String postId) {
|
||||
imageRepository.stopByPostIdAndIsNotCompleted(postId);
|
||||
}
|
||||
|
||||
public Optional<Queued> findQueuedByThreadId(String threadId) {
|
||||
return queuedRepository.findByThreadId(threadId);
|
||||
}
|
||||
|
||||
public List<Queued> findAllQueued() {
|
||||
return queuedRepository.findAll();
|
||||
}
|
||||
|
||||
public Optional<Post> findById(Long aLong) {
|
||||
return postRepository.findById(aLong);
|
||||
}
|
||||
|
||||
public Optional<Image> findImageById(Long aLong) {
|
||||
return imageRepository.findById(aLong);
|
||||
}
|
||||
|
||||
public Optional<Queued> findQueuedById(Long aLong) {
|
||||
return queuedRepository.findById(aLong);
|
||||
}
|
||||
|
||||
public synchronized void setMetadata(Post post, Metadata metadata) {
|
||||
if (metadataRepository.findByPostId(post.getPostId()).isEmpty()) {
|
||||
metadata.setPostIdRef(post.getId());
|
||||
metadataRepository.save(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
public Optional<Metadata> findMetadataByPostId(String postId) {
|
||||
return metadataRepository.findByPostId(postId);
|
||||
}
|
||||
|
||||
public void updateImageStatus(Status status, Long id) {
|
||||
imageRepository.updateStatus(status, id);
|
||||
}
|
||||
|
||||
public void updateImageCurrent(long current, Long id) {
|
||||
imageRepository.updateCurrent(current, id);
|
||||
}
|
||||
|
||||
public void updateImageTotal(long total, Long id) {
|
||||
imageRepository.updateTotal(total, id);
|
||||
}
|
||||
|
||||
public void updatePostStatus(Status status, Long id) {
|
||||
postRepository.updateStatus(status, id);
|
||||
}
|
||||
|
||||
public void updateDownloadDirectory(String postFolderName, Long id) {
|
||||
postRepository.updateFolderName(postFolderName, id);
|
||||
}
|
||||
|
||||
public void updatePostTitle(String title, Long id) {
|
||||
postRepository.updateTitle(title, id);
|
||||
}
|
||||
|
||||
public void updatePostThanked(boolean thanked, Long id) {
|
||||
postRepository.updateThanked(thanked, id);
|
||||
}
|
||||
|
||||
public Optional<Event> findEventById(Long id) {
|
||||
return eventRepository.findById(id);
|
||||
}
|
||||
|
||||
public List<Event> findAllEvents() {
|
||||
return eventRepository.findAll();
|
||||
}
|
||||
|
||||
public void clearQueueLinks() {
|
||||
queuedRepository.deleteAll();
|
||||
public Optional<Metadata> findMetadataByPostId(String postId) {
|
||||
return metadataRepository.findByPostId(postId);
|
||||
}
|
||||
|
||||
public void updateImageStatus(Status status, Long id) {
|
||||
imageRepository.updateStatus(status, id);
|
||||
}
|
||||
|
||||
public void updateImageCurrent(long current, Long id) {
|
||||
imageRepository.updateCurrent(current, id);
|
||||
}
|
||||
|
||||
public void updateImageTotal(long total, Long id) {
|
||||
imageRepository.updateTotal(total, id);
|
||||
}
|
||||
|
||||
public Optional<LogEvent> findEventById(Long id) {
|
||||
return eventRepository.findById(id);
|
||||
}
|
||||
|
||||
public List<LogEvent> findAllEvents() {
|
||||
return eventRepository.findAll();
|
||||
}
|
||||
|
||||
public void clearQueueLinks() {
|
||||
queuedRepository.deleteAll();
|
||||
}
|
||||
|
||||
private synchronized void sortPostsByRank() {
|
||||
List<Post> posts = findAllPosts();
|
||||
posts.sort(Comparator.comparing(Post::getAddedOn));
|
||||
for (int i = 0; i < posts.size(); i++) {
|
||||
posts.get(i).setRank(i);
|
||||
updatePostRank(i, posts.get(i).getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,46 +4,38 @@ import lombok.Getter;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
public class DownloadSpeedService {
|
||||
|
||||
private final AtomicLong read = new AtomicLong(0);
|
||||
private final Sinks.Many<Long> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
@Getter
|
||||
private long currentValue;
|
||||
private boolean allowWrite = false;
|
||||
private final AtomicLong read = new AtomicLong(0);
|
||||
private final EventBus eventBus;
|
||||
@Getter private long currentValue;
|
||||
private boolean allowWrite = false;
|
||||
|
||||
public Flux<Long> getReadBytesPerSecond() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
public DownloadSpeedService(EventBus eventBus) {
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
public void increase(long read) {
|
||||
if (allowWrite) {
|
||||
this.read.addAndGet(read);
|
||||
}
|
||||
public void increase(long read) {
|
||||
if (allowWrite) {
|
||||
this.read.addAndGet(read);
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 1000)
|
||||
private void calc() {
|
||||
allowWrite = false;
|
||||
long newValue = read.getAndSet(0);
|
||||
if (newValue != currentValue) {
|
||||
currentValue = newValue;
|
||||
sink.emitNext(currentValue, EmitHandler.RETRY);
|
||||
}
|
||||
allowWrite = true;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
@Scheduled(fixedDelay = 1000)
|
||||
private void calc() {
|
||||
allowWrite = false;
|
||||
long newValue = read.getAndSet(0);
|
||||
if (newValue != currentValue) {
|
||||
currentValue = newValue;
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.BYTES_PER_SECOND, currentValue));
|
||||
}
|
||||
allowWrite = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,51 +5,38 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import tn.mnlr.vripper.download.DownloadService;
|
||||
import tn.mnlr.vripper.download.PendingQueue;
|
||||
import tn.mnlr.vripper.listener.EmitHandler;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.services.domain.GlobalState;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
public class GlobalStateService {
|
||||
|
||||
private final PendingQueue pendingQueue;
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final Sinks.Many<GlobalState> sink = Sinks.many().multicast().onBackpressureBuffer();
|
||||
@Getter
|
||||
private GlobalState currentState;
|
||||
private final DownloadService downloadService;
|
||||
private final DataService dataService;
|
||||
private final EventBus eventBus;
|
||||
@Getter private GlobalState currentState;
|
||||
|
||||
@Autowired
|
||||
public GlobalStateService(PendingQueue pendingQueue, DownloadService downloadService, DataService dataService) {
|
||||
this.pendingQueue = pendingQueue;
|
||||
this.downloadService = downloadService;
|
||||
this.dataService = dataService;
|
||||
}
|
||||
@Autowired
|
||||
public GlobalStateService(
|
||||
DownloadService downloadService, DataService dataService, EventBus eventBus) {
|
||||
this.downloadService = downloadService;
|
||||
this.dataService = dataService;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
public Flux<GlobalState> getGlobalState() {
|
||||
return sink.asFlux();
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 3000)
|
||||
private void interval() {
|
||||
GlobalState newGlobalState = new GlobalState(
|
||||
downloadService.runningCount(),
|
||||
pendingQueue.size(),
|
||||
dataService.countErrorImages());
|
||||
if (!newGlobalState.equals(currentState)) {
|
||||
currentState = newGlobalState;
|
||||
sink.emitNext(currentState, EmitHandler.RETRY);
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
sink.emitComplete(EmitHandler.RETRY);
|
||||
@Scheduled(fixedDelay = 3000)
|
||||
private void interval() {
|
||||
GlobalState newGlobalState =
|
||||
new GlobalState(
|
||||
downloadService.runningCount(),
|
||||
downloadService.pendingCount(),
|
||||
dataService.countErrorImages());
|
||||
if (!newGlobalState.equals(currentState)) {
|
||||
currentState = newGlobalState;
|
||||
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,83 +22,92 @@ import java.net.URISyntaxException;
|
||||
@Slf4j
|
||||
public class HostService {
|
||||
|
||||
private final ConnectionService cm;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final ConnectionService cm;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
|
||||
@Autowired
|
||||
public HostService(ConnectionService cm, HtmlProcessorService htmlProcessorService) {
|
||||
this.cm = cm;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
@Autowired
|
||||
public HostService(ConnectionService cm, HtmlProcessorService htmlProcessorService) {
|
||||
this.cm = cm;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
}
|
||||
|
||||
public Response getResponse(final String url, final HttpClientContext context)
|
||||
throws HostException {
|
||||
String basePage;
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url, context);
|
||||
Header[] headers;
|
||||
log.debug(String.format("Requesting %s", url));
|
||||
try (CloseableHttpResponse response =
|
||||
(CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new HostException(
|
||||
String.format(
|
||||
"Unexpected response code: %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
headers = response.getAllHeaders();
|
||||
basePage = EntityUtils.toString(response.getEntity());
|
||||
log.debug(String.format("%s response: %n%s", url, basePage));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
|
||||
public Response getResponse(final String url, final HttpClientContext context) throws HostException {
|
||||
String basePage;
|
||||
try {
|
||||
log.debug(String.format("Cleaning %s response", url));
|
||||
return new Response(htmlProcessorService.clean(basePage), headers);
|
||||
} catch (HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
}
|
||||
|
||||
HttpClient client = cm.getClient().build();
|
||||
HttpGet httpGet = cm.buildHttpGet(url, context);
|
||||
Header[] headers;
|
||||
log.debug(String.format("Requesting %s", url));
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
|
||||
}
|
||||
headers = response.getAllHeaders();
|
||||
basePage = EntityUtils.toString(response.getEntity());
|
||||
log.debug(String.format("%s response: %n%s", url, basePage));
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
} catch (IOException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
public String appendUri(String uri, String appendQuery) throws URISyntaxException {
|
||||
URI oldUri = new URI(uri);
|
||||
|
||||
try {
|
||||
log.debug(String.format("Cleaning %s response", url));
|
||||
return new Response(htmlProcessorService.clean(basePage), headers);
|
||||
} catch (HtmlProcessorException e) {
|
||||
throw new HostException(e);
|
||||
}
|
||||
String newQuery = oldUri.getQuery();
|
||||
if (newQuery == null) {
|
||||
newQuery = appendQuery;
|
||||
} else {
|
||||
newQuery += "&" + appendQuery;
|
||||
}
|
||||
|
||||
public String appendUri(String uri, String appendQuery) throws URISyntaxException {
|
||||
URI oldUri = new URI(uri);
|
||||
return new URI(
|
||||
oldUri.getScheme(),
|
||||
oldUri.getAuthority(),
|
||||
oldUri.getPath(),
|
||||
newQuery,
|
||||
oldUri.getFragment())
|
||||
.toString();
|
||||
}
|
||||
|
||||
String newQuery = oldUri.getQuery();
|
||||
if (newQuery == null) {
|
||||
newQuery = appendQuery;
|
||||
} else {
|
||||
newQuery += "&" + appendQuery;
|
||||
}
|
||||
public String getDefaultImageName(final String imgUrl) {
|
||||
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
|
||||
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
|
||||
return imgUrl;
|
||||
}
|
||||
|
||||
return new URI(oldUri.getScheme(), oldUri.getAuthority(),
|
||||
oldUri.getPath(), newQuery, oldUri.getFragment()).toString();
|
||||
@Getter
|
||||
public static class Response {
|
||||
|
||||
private final Document document;
|
||||
private final Header[] headers;
|
||||
|
||||
protected Response(Document document, Header[] headers) {
|
||||
this.document = document;
|
||||
this.headers = headers;
|
||||
}
|
||||
}
|
||||
|
||||
public String getDefaultImageName(final String imgUrl) {
|
||||
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
|
||||
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
|
||||
return imgUrl;
|
||||
}
|
||||
@Getter
|
||||
public static class NameUrl {
|
||||
|
||||
@Getter
|
||||
public static class Response {
|
||||
private final String name;
|
||||
private final String url;
|
||||
|
||||
private final Document document;
|
||||
private final Header[] headers;
|
||||
|
||||
protected Response(Document document, Header[] headers) {
|
||||
this.document = document;
|
||||
this.headers = headers;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class NameUrl {
|
||||
|
||||
private final String name;
|
||||
private final String url;
|
||||
|
||||
public NameUrl(String name, String url) {
|
||||
this.name = name;
|
||||
this.url = url;
|
||||
}
|
||||
public NameUrl(String name, String url) {
|
||||
this.name = name;
|
||||
this.url = url;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,12 +11,12 @@ import tn.mnlr.vripper.exception.HtmlProcessorException;
|
||||
@Service
|
||||
public class HtmlProcessorService {
|
||||
|
||||
public Document clean(String htmlContent) throws HtmlProcessorException {
|
||||
try {
|
||||
TagNode clean = new HtmlCleaner().clean(htmlContent);
|
||||
return new DomSerializer(new CleanerProperties()).createDOM(clean);
|
||||
} catch (Exception e) {
|
||||
throw new HtmlProcessorException(e);
|
||||
}
|
||||
public Document clean(String htmlContent) throws HtmlProcessorException {
|
||||
try {
|
||||
TagNode clean = new HtmlCleaner().clean(htmlContent);
|
||||
return new DomSerializer(new CleanerProperties()).createDOM(clean);
|
||||
} catch (Exception e) {
|
||||
throw new HtmlProcessorException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,177 +1,50 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.jodah.failsafe.Failsafe;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import tn.mnlr.vripper.exception.DownloadException;
|
||||
import tn.mnlr.vripper.exception.PostParseException;
|
||||
import tn.mnlr.vripper.jpa.domain.Metadata;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable;
|
||||
import tn.mnlr.vripper.tasks.MetadataRunnable;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MetadataService {
|
||||
|
||||
private static final List<String> dictionary = Arrays.asList("download", "link", "rapidgator", "filefactory", "filefox");
|
||||
private final LoadingCache<Key, Metadata> cache;
|
||||
private final ConnectionService cm;
|
||||
private final VGAuthService VGAuthService;
|
||||
private final HtmlProcessorService htmlProcessorService;
|
||||
private final XpathService xpathService;
|
||||
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
|
||||
private final ThreadPoolService threadPoolService;
|
||||
|
||||
public MetadataService(ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService, ThreadPoolService threadPoolService) {
|
||||
this.cm = cm;
|
||||
this.VGAuthService = VGAuthService;
|
||||
this.htmlProcessorService = htmlProcessorService;
|
||||
this.xpathService = xpathService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
public MetadataService(ThreadPoolService threadPoolService) {
|
||||
this.threadPoolService = threadPoolService;
|
||||
}
|
||||
|
||||
CacheLoader<Key, Metadata> loader = new CacheLoader<>() {
|
||||
@Override
|
||||
public Metadata load(@NonNull Key key) {
|
||||
return fetchMetadata(key);
|
||||
}
|
||||
};
|
||||
cache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(30, TimeUnit.MINUTES)
|
||||
.build(loader);
|
||||
public void startFetchingMetadata(Post post) {
|
||||
MetadataRunnable runnable = new MetadataRunnable(post);
|
||||
threadPoolService.getGeneralExecutor().submit(runnable);
|
||||
fetchingMetadata.put(post.getPostId(), runnable);
|
||||
}
|
||||
|
||||
public void stopFetchingMetadata(List<String> postIds) {
|
||||
List<MetadataRunnable> stopping = new ArrayList<>();
|
||||
|
||||
for (Map.Entry<String, MetadataRunnable> entry : this.fetchingMetadata.entrySet()) {
|
||||
if (postIds.contains(entry.getKey())) {
|
||||
stopping.add(entry.getValue());
|
||||
entry.getValue().stop();
|
||||
}
|
||||
}
|
||||
|
||||
public Metadata get(Post post) throws ExecutionException {
|
||||
Key key = new Key(post.getPostId(), post.getThreadId(), post.getUrl());
|
||||
return Metadata.from(cache.get(key));
|
||||
}
|
||||
|
||||
public void startFetchingMetadata(Post post) {
|
||||
MetadataRunnable runnable = new MetadataRunnable(post);
|
||||
threadPoolService.getGeneralExecutor().submit(runnable);
|
||||
fetchingMetadata.put(post.getPostId(), runnable);
|
||||
}
|
||||
|
||||
public void stopFetchingMetadata(Post post) {
|
||||
this.fetchingMetadata.forEach((k, v) -> {
|
||||
if (k.equals(post.getPostId())) {
|
||||
v.setInterrupted(true);
|
||||
}
|
||||
});
|
||||
fetchingMetadata.remove(post.getPostId());
|
||||
}
|
||||
|
||||
private Metadata fetchMetadata(Key key) {
|
||||
HttpGet httpGet = cm.buildHttpGet(key.getUrl(), null);
|
||||
Metadata metadata = new Metadata();
|
||||
Failsafe.with(cm.getRetryPolicy())
|
||||
.onFailure(e -> {
|
||||
if (e.getFailure() instanceof InterruptedException || e.getFailure().getCause() instanceof InterruptedException) {
|
||||
log.debug("Fetching interrupted");
|
||||
return;
|
||||
}
|
||||
log.error(String.format("Error occurred when getting post metadata, postId %s", key.getPostId()), e.getFailure());
|
||||
})
|
||||
.run(() -> {
|
||||
HttpClient connection = cm.getClient().build();
|
||||
try (CloseableHttpResponse response = (CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
throw new DownloadException(String.format("Unexpected response code '%d' for %s", response.getStatusLine().getStatusCode(), httpGet));
|
||||
}
|
||||
try {
|
||||
if (Thread.interrupted()) {
|
||||
return;
|
||||
}
|
||||
Document document = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
|
||||
Node postNode = xpathService.getAsNode(document, String.format("//li[@id='post_%s']/div[contains(@class, 'postdetails')]", key.getPostId()));
|
||||
String postedBy = xpathService.getAsNode(postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font").getTextContent().trim();
|
||||
metadata.setPostedBy(postedBy);
|
||||
|
||||
Node node = xpathService.getAsNode(document, String.format("//div[@id='post_message_%s']", key.getPostId()));
|
||||
metadata.setResolvedNames(findTitleInContent(node));
|
||||
|
||||
metadata.setPostId(key.getPostId());
|
||||
} catch (Exception e) {
|
||||
throw new PostParseException(String.format("Failed to parse thread %s, post %s", key.getThreadId(), key.getPostId()), e);
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.getEntity());
|
||||
}
|
||||
}
|
||||
});
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private List<String> findTitleInContent(Node node) {
|
||||
List<String> altTitle = new ArrayList<>();
|
||||
findTitle(node, altTitle, new AtomicBoolean(true));
|
||||
return altTitle.stream().distinct().collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private void findTitle(Node node, List<String> altTitle, AtomicBoolean keepGoing) {
|
||||
if (!keepGoing.get()) {
|
||||
return;
|
||||
}
|
||||
if (node.getNodeName().equals("a") || node.getNodeName().equals("img")) {
|
||||
keepGoing.set(false);
|
||||
return;
|
||||
}
|
||||
if (node.getNodeType() == Node.ELEMENT_NODE) {
|
||||
for (int i = 0; i < node.getChildNodes().getLength(); i++) {
|
||||
Node item = node.getChildNodes().item(i);
|
||||
findTitle(item, altTitle, keepGoing);
|
||||
if (!keepGoing.get()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
} else if (node.getNodeType() == Node.TEXT_NODE) {
|
||||
String text = node.getTextContent().trim();
|
||||
if (!text.isBlank() && dictionary.stream().noneMatch(e -> text.toLowerCase().contains(e.toLowerCase()))) {
|
||||
altTitle.add(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
static class Key {
|
||||
private final String postId;
|
||||
private final String threadId;
|
||||
private final String url;
|
||||
|
||||
Key(String postId, String threadId, String url) {
|
||||
this.postId = postId;
|
||||
this.threadId = threadId;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
Key key = (Key) o;
|
||||
return Objects.equals(postId, key.postId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(postId);
|
||||
}
|
||||
while (!stopping.isEmpty()) {
|
||||
stopping.removeIf(MetadataRunnable::isFinished);
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
postIds.forEach(fetchingMetadata::remove);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.exception.RenameException;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import tn.mnlr.vripper.services.domain.Settings;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@@ -17,93 +18,109 @@ import java.util.concurrent.locks.ReentrantLock;
|
||||
@Slf4j
|
||||
public class PathService {
|
||||
|
||||
private final SettingsService settingsService;
|
||||
private final DataService dataService;
|
||||
private final DataService dataService;
|
||||
|
||||
@Getter
|
||||
private final ReentrantLock directoryAccess = new ReentrantLock();
|
||||
@Getter private final ReentrantLock directoryAccess = new ReentrantLock();
|
||||
|
||||
public PathService(SettingsService settingsService, DataService dataService) {
|
||||
this.settingsService = settingsService;
|
||||
this.dataService = dataService;
|
||||
public PathService(DataService dataService) {
|
||||
this.dataService = dataService;
|
||||
}
|
||||
|
||||
public final File calcDownloadDirectory(Post post, Settings settings) {
|
||||
return new File(settings.getDownloadPath(), post.getDownloadDirectory());
|
||||
}
|
||||
|
||||
private File getRootFolder(
|
||||
@NonNull String forum, @NonNull String threadTitle, Settings settings) {
|
||||
File sourceFolder =
|
||||
settings.getSubLocation()
|
||||
? new File(settings.getDownloadPath(), sanitize(forum))
|
||||
: new File(settings.getDownloadPath());
|
||||
return settings.getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
|
||||
}
|
||||
|
||||
public final void createDefaultPostFolder(Post post, Settings settings) {
|
||||
File downloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
|
||||
downloadDirectory =
|
||||
new File(
|
||||
downloadDirectory,
|
||||
settings.getAppendPostId()
|
||||
? sanitize(post.getTitle()) + "_" + post.getPostId()
|
||||
: sanitize(post.getTitle()));
|
||||
downloadDirectory = makeDir(downloadDirectory);
|
||||
post.setDownloadDirectory(
|
||||
downloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
|
||||
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
}
|
||||
|
||||
public final void rename(@NonNull String postId, @NonNull String altName, Settings settings)
|
||||
throws RenameException {
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (altName.equals(post.getTitle())) {
|
||||
return;
|
||||
}
|
||||
|
||||
public final File calcDownloadDirectory(Post post) {
|
||||
return new File(settingsService.getSettings().getDownloadPath(), post.getDownloadDirectory());
|
||||
// Download have not started yet
|
||||
if (post.getDownloadDirectory() == null) {
|
||||
post.setTitle(altName);
|
||||
dataService.updatePostTitle(post.getTitle(), post.getId());
|
||||
return;
|
||||
}
|
||||
|
||||
private File getRootFolder(@NonNull String forum, @NonNull String threadTitle) {
|
||||
File sourceFolder = settingsService.getSettings().getSubLocation() ? new File(settingsService.getSettings().getDownloadPath(), sanitize(forum)) : new File(settingsService.getSettings().getDownloadPath());
|
||||
return settingsService.getSettings().getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
|
||||
File newDownloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
|
||||
newDownloadDirectory = new File(newDownloadDirectory, sanitize(altName));
|
||||
File currentDownloadDirectory = calcDownloadDirectory(post, settings);
|
||||
try {
|
||||
directoryAccess.lock();
|
||||
Files.move(
|
||||
currentDownloadDirectory.toPath(),
|
||||
newDownloadDirectory.toPath(),
|
||||
StandardCopyOption.ATOMIC_MOVE);
|
||||
post.setDownloadDirectory(
|
||||
newDownloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
|
||||
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
|
||||
post.setTitle(altName);
|
||||
dataService.updatePostTitle(post.getTitle(), post.getId());
|
||||
} catch (IOException e) {
|
||||
throw new RenameException(
|
||||
String.format(
|
||||
"Failed to move files from %s to %s", currentDownloadDirectory, newDownloadDirectory),
|
||||
e);
|
||||
} finally {
|
||||
directoryAccess.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private File makeDir(@NonNull final File sourceFolder) {
|
||||
int counter = 1;
|
||||
File folder = sourceFolder;
|
||||
|
||||
while (folder.exists()) {
|
||||
folder = new File(sourceFolder.toString() + '.' + counter++);
|
||||
}
|
||||
|
||||
public final void createDefaultPostFolder(Post post) {
|
||||
File downloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle());
|
||||
downloadDirectory = new File(downloadDirectory, settingsService.getSettings().getAppendPostId() ? sanitize(post.getTitle()) + "_" + post.getPostId() : sanitize(post.getTitle()));
|
||||
downloadDirectory = makeDir(downloadDirectory);
|
||||
post.setDownloadDirectory(downloadDirectory.getAbsolutePath().replace(settingsService.getSettings().getDownloadPath(), ""));
|
||||
dataService.updateDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
if (!folder.mkdirs()) {
|
||||
throw new RuntimeException(String.format("Failed to create the folder %s", sourceFolder));
|
||||
}
|
||||
|
||||
public final void rename(@NonNull String postId, @NonNull String altName) throws RenameException {
|
||||
Post post = dataService.findPostByPostId(postId).orElseThrow();
|
||||
if (altName.equals(post.getTitle())) {
|
||||
return;
|
||||
}
|
||||
return folder;
|
||||
}
|
||||
|
||||
// Download have not started yet
|
||||
if (post.getDownloadDirectory() == null) {
|
||||
post.setTitle(altName);
|
||||
dataService.updatePostTitle(post.getTitle(), post.getId());
|
||||
return;
|
||||
}
|
||||
private String sanitize(final String folderName) {
|
||||
String sanitizedFolderName =
|
||||
folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
|
||||
log.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
|
||||
return sanitizedFolderName;
|
||||
}
|
||||
|
||||
File newDownloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle());
|
||||
newDownloadDirectory = new File(newDownloadDirectory, sanitize(altName));
|
||||
File currentDownloadDirectory = calcDownloadDirectory(post);
|
||||
try {
|
||||
directoryAccess.lock();
|
||||
Files.move(currentDownloadDirectory.toPath(), newDownloadDirectory.toPath(), StandardCopyOption.ATOMIC_MOVE);
|
||||
post.setDownloadDirectory(newDownloadDirectory.getAbsolutePath().replace(settingsService.getSettings().getDownloadPath(), ""));
|
||||
dataService.updateDownloadDirectory(post.getDownloadDirectory(), post.getId());
|
||||
|
||||
post.setTitle(altName);
|
||||
dataService.updatePostTitle(post.getTitle(), post.getId());
|
||||
} catch (IOException e) {
|
||||
throw new RenameException(String.format("Failed to move files from %s to %s", currentDownloadDirectory.toString(), newDownloadDirectory.toString()), e);
|
||||
} finally {
|
||||
directoryAccess.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private File makeDir(@NonNull final File sourceFolder) {
|
||||
int counter = 1;
|
||||
File folder = sourceFolder;
|
||||
|
||||
while (folder.exists()) {
|
||||
folder = new File(sourceFolder.toString() + '.' + counter++);
|
||||
}
|
||||
|
||||
if (!folder.mkdirs()) {
|
||||
throw new RuntimeException(String.format("Failed to create the folder %s", sourceFolder.toString()));
|
||||
}
|
||||
|
||||
return folder;
|
||||
}
|
||||
|
||||
private String sanitize(final String folderName) {
|
||||
String sanitizedFolderName = folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
|
||||
log.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
|
||||
return sanitizedFolderName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Will sanitize the image name and remove extension
|
||||
*
|
||||
* @param imageName path string
|
||||
* @return Sanitized local path string
|
||||
*/
|
||||
public final String formatImageFileName(String imageName) {
|
||||
return sanitize(imageName);
|
||||
}
|
||||
/**
|
||||
* Will sanitize the image name and remove extension
|
||||
*
|
||||
* @param imageName path string
|
||||
* @return Sanitized local path string
|
||||
*/
|
||||
public final String formatImageFileName(String imageName) {
|
||||
return sanitize(imageName);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
package tn.mnlr.vripper.services;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tn.mnlr.vripper.jpa.domain.Post;
|
||||
import reactor.core.Disposable;
|
||||
import tn.mnlr.vripper.event.Event;
|
||||
import tn.mnlr.vripper.event.EventBus;
|
||||
import tn.mnlr.vripper.jpa.domain.Queued;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanParser;
|
||||
import tn.mnlr.vripper.services.domain.MultiPostScanResult;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddPostRunnable;
|
||||
import tn.mnlr.vripper.tasks.AddQueuedRunnable;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -22,48 +23,51 @@ import java.util.concurrent.TimeUnit;
|
||||
@Slf4j
|
||||
public class PostService {
|
||||
|
||||
private final DataService dataService;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final MetadataService metadataService;
|
||||
private final LoadingCache<Queued, MultiPostScanResult> cache;
|
||||
private final DataService dataService;
|
||||
private final ThreadPoolService threadPoolService;
|
||||
private final LoadingCache<Queued, MultiPostScanResult> cache;
|
||||
private final Disposable disposable;
|
||||
|
||||
@Autowired
|
||||
public PostService(DataService dataService, ThreadPoolService threadPoolService, MetadataService metadataService) {
|
||||
this.dataService = dataService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
this.metadataService = metadataService;
|
||||
@Autowired
|
||||
public PostService(
|
||||
DataService dataService, ThreadPoolService threadPoolService, EventBus eventBus) {
|
||||
this.dataService = dataService;
|
||||
this.threadPoolService = threadPoolService;
|
||||
cache =
|
||||
Caffeine.newBuilder()
|
||||
.expireAfterWrite(5, TimeUnit.MINUTES)
|
||||
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
|
||||
disposable =
|
||||
eventBus
|
||||
.flux()
|
||||
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
|
||||
.subscribe(e -> this.cache.invalidateAll());
|
||||
}
|
||||
|
||||
CacheLoader<Queued, MultiPostScanResult> loader = new CacheLoader<>() {
|
||||
@Override
|
||||
public MultiPostScanResult load(@NonNull Queued multiPostItem) throws Exception {
|
||||
return new MultiPostScanParser(multiPostItem).parse();
|
||||
}
|
||||
};
|
||||
|
||||
cache = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(30, TimeUnit.MINUTES)
|
||||
.build(loader);
|
||||
@PreDestroy
|
||||
private void destroy() {
|
||||
if (disposable != null) {
|
||||
disposable.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public void stopFetchingMetadata(Post post) {
|
||||
metadataService.stopFetchingMetadata(post);
|
||||
public void processMultiPost(List<Queued> queuedList) {
|
||||
for (Queued queued : queuedList) {
|
||||
if (queued.getPostId() != null) {
|
||||
threadPoolService
|
||||
.getGeneralExecutor()
|
||||
.submit(new AddPostRunnable(queued.getPostId(), queued.getThreadId()));
|
||||
} else {
|
||||
threadPoolService.getGeneralExecutor().submit(new AddQueuedRunnable(queued));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void processMultiPost(List<Queued> queuedList) {
|
||||
for (Queued queued : queuedList) {
|
||||
if (queued.getPostId() != null) {
|
||||
threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(queued.getPostId(), queued.getThreadId()));
|
||||
} else {
|
||||
threadPoolService.getGeneralExecutor().submit(new AddQueuedRunnable(queued));
|
||||
}
|
||||
}
|
||||
}
|
||||
public MultiPostScanResult get(Queued queued) throws ExecutionException {
|
||||
return cache.get(queued);
|
||||
}
|
||||
|
||||
public MultiPostScanResult get(Queued queued) throws ExecutionException {
|
||||
return cache.get(queued);
|
||||
}
|
||||
|
||||
public void remove(String threadId) {
|
||||
dataService.removeQueueLink(threadId);
|
||||
}
|
||||
public void remove(String threadId) {
|
||||
dataService.removeQueueLink(threadId);
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user