Compare commits

...
171 Commits
Author SHA1 Message Date
death-claw 42e1aa73e2 v3.2.1 2021-01-12 20:02:17 +01:00
death-claw 26fde22af2 Code reformat 2021-01-12 19:37:12 +01:00
death-claw 798f39d54f Add settings for connection timeout
Add settings for maximum attempts
Enhance posts rename logic
Dependencies update
Switch reactive from rx-java to reactor
2021-01-12 19:32:43 +01:00
death-claw 3a21dfac38 Remove unused font 2021-01-06 14:55:49 +01:00
death-claw e8355b88b9 v3.2 2020-11-16 19:40:36 +01:00
death-claw 379660236d Commit missing files 2020-11-16 19:31:59 +01:00
death-claw c86ee91c27 v3.1.0 2020-11-16 16:44:26 +01:00
death-claw ff9d253242 UI revamp
Fix bugs
2020-11-16 16:42:30 +01:00
death-claw 53b45e00ba v3.0.11 2020-10-31 15:09:19 +01:00
death-claw 247b12a525 Add support for vipr.im 2020-10-31 15:06:26 +01:00
death-claw 7a8d332e6e v3.0.10 2020-10-04 16:31:57 +01:00
death-claw 749a6f18d2 Fix bug on settings update 2020-10-04 16:27:26 +01:00
death-claw 4107048f12 v3.0.9 2020-09-27 10:14:08 +01:00
death-claw f5d9ab85a2 Revert to appending post id to download folders
Fix a bug that lead to adding duplicate posts
Remove old unused settings
2020-09-27 10:12:29 +01:00
death-claw b82ee7b494 v3.0.8 2020-08-22 19:01:08 +01:00
death-claw 8a31dfc012 Bug fix which breaks concurrent downloads 2020-08-22 18:59:44 +01:00
death-claw 734c1b72c7 v3.0.7 2020-08-22 16:45:43 +01:00
death-claw 7abb7df1f6 Fix MacOs build 2020-08-22 16:44:24 +01:00
death-claw 0a7fa99a64 v3.0.6 2020-08-18 08:33:57 +01:00
death-claw 691026c8a9 Fix TurboImageHost 2020-08-18 08:32:33 +01:00
death-claw 3ef004b947 v3.0.5 2020-08-17 20:06:12 +01:00
death-claw 1b22a93e28 Update packages
Host class refactoring
2020-08-17 20:04:39 +01:00
death-claw 2639df0aaa Update packages
Host class refactoring
2020-08-17 20:04:23 +01:00
death-claw d5ae5d57d1 v3.0.4 2020-08-16 16:11:11 +01:00
death-claw 537d76d7be Improve rename UI 2020-08-16 16:06:17 +01:00
death-claw c1247faacc v3.0.3 2020-08-16 11:29:01 +01:00
death-claw efcc685f30 Remove notification feature 2020-08-16 11:26:57 +01:00
death-claw c825b22ce4 Fix some bugs with MacOS 2020-08-16 10:58:19 +01:00
death-claw 6ef5d5474f Fix some bugs with MacOS 2020-08-16 10:20:56 +01:00
death-claw bd25801c6c Fix base dir configuration for MacOS 2020-08-16 00:18:18 +01:00
death-claw 377b93c1ac Fix base dir configuration for AppImage 2020-08-16 00:02:18 +01:00
death-claw d498d20f6b Set default download path to user's home 2020-08-15 23:23:43 +01:00
death-claw d18544d414 Fix bin location for mac 2020-08-15 23:10:58 +01:00
death-claw 3c872b79bd Update app data configuration 2020-08-15 23:00:48 +01:00
death-claw 37900cac49 v3.0.2 2020-08-15 15:38:53 +01:00
death-claw be57c76d93 Fix bugs with download queue 2020-08-15 15:36:47 +01:00
death-claw 8b52ad7858 v3.0.1 2020-08-03 18:48:45 +01:00
death-claw a0976bde02 Fix bugs with download queue 2020-08-03 18:43:42 +01:00
death-claw 15c97d5cfc v3.0.0 2020-08-03 15:06:36 +01:00
death-claw 905cbd5a35 Code refactoring
Fix bugs with rename gallery
2020-08-03 15:03:42 +01:00
death-clawandGitHub 5c0e2e6185 Merge pull request #12 from life-claw/imgVenueFix
Fixed ImageVenue
2020-07-30 20:56:41 +01:00
death-clawandGitHub 4836d2dacb Merge branch 'master' into imgVenueFix 2020-07-30 20:56:30 +01:00
death-claw bc81ec1ab5 Bug fixes
Code enhancements
2020-07-30 14:59:58 +01:00
death-claw 8a4d039a37 Bug fixes
Code enhancements
2020-07-30 14:00:56 +01:00
death-claw 468639264a Major rewrites of data persistence, websocket, and user interface 2020-07-26 15:10:09 +01:00
death-claw 9b649b470f Major rewrites of data persistence, websocket, and user interface 2020-07-26 14:45:04 +01:00
life-claw edd5cf2bf7 Fixed ImageVenue
Updated XPaths to match their current markup
And added HTTPs redirect, since they support that now
2020-07-17 16:57:51 -05:00
death-claw 67c605e640 v2.12.1 2020-05-22 15:55:05 +01:00
death-claw 8d4b878656 Fix alternative name bug 2020-05-22 15:54:11 +01:00
death-claw bf532dd1d0 v2.12.0 2020-05-21 16:43:03 +01:00
death-claw eb1a125e59 Use new user hash from the api 2020-05-21 16:41:16 +01:00
death-claw 256659a72d v2.11.7 2020-05-08 12:57:41 +01:00
death-claw dbf3aa7e7d Fix bug with paths creation
Move Open Link button
Rename rename all title
2020-05-08 12:56:11 +01:00
death-claw a75ce181ff v2.11.6 2020-05-04 21:07:16 +01:00
death-claw 37ae6b4ebf Merge branch 'master' of https://github.com/death-claw/vripper-project 2020-05-04 21:06:48 +01:00
death-claw 6631a6a95b PNG filenames are now correctly named
Fix issue with special characters in post titles
2020-05-04 21:04:51 +01:00
death-clawandGitHub 918f8578ad Update CHANGELOG.md 2020-05-03 16:40:26 +01:00
death-claw 10fa9f68ae v2.11.5 2020-05-03 15:18:28 +01:00
death-claw 6b5c9ccb0f Reduce rows buffer size 2020-05-03 15:16:31 +01:00
death-claw 652e8f1a9d v2.11.4 2020-05-03 14:24:27 +01:00
death-claw fe53ae733d Code cleanup
Dependencies update
2020-05-03 14:22:40 +01:00
death-claw 690dab2ae8 v2.11.3 2020-04-30 18:26:42 +01:00
death-claw 530542163e Add missing files 2020-04-30 18:26:23 +01:00
death-claw cf7f4c75c0 Source code restructure
UI fixes and enhancements
Add menu entry to rename a gallery to first alternative title
Add menu on toolbar for massive rename to first alternative title
2020-04-30 18:23:11 +01:00
death-claw c6b395cf4c v2.11.2 2020-04-26 22:06:34 +01:00
death-claw e72edeed62 Fix icon
Bug fixes
2020-04-26 22:02:51 +01:00
death-claw 3f22d53f89 v2.11.1 2020-04-25 16:32:25 +01:00
death-claw 08c0fa62b1 Add option to rename gallery
Code source maintenance
2020-04-25 16:28:56 +01:00
death-claw 4c79cca079 v2.11.0 2020-03-29 17:37:28 +01:00
death-clawandGitHub 67096242a2 Merge pull request #4 from life-claw/pixroute
Added Support for pixroute.com
2020-03-29 17:30:34 +01:00
death-claw f66112375d Merge branch 'master' of https://github.com/death-claw/vripper-project 2020-03-29 17:28:39 +01:00
death-claw c5a082e248 Upgrade Electron to v8
Upgrade Angular to v9
App settings are now portable in a json file
Add grab post title from thread option
General code source maintenance
2020-03-29 17:03:14 +01:00
life-claw c2a1b0aa33 Added Support for pixroute.com 2020-03-08 13:25:57 -05:00
death-clawandGitHub a89dcc96d8 Update README.md 2020-02-11 21:16:37 +01:00
death-claw b2a10eafd9 v2.10.8 2020-01-25 16:00:05 +01:00
death-claw f027aff89d Fix app url 2020-01-25 15:57:45 +01:00
death-claw b4eaf0d1c4 v2.10.7 2020-01-25 14:45:54 +01:00
death-claw 9d2975faaa Drop SockJs and switch to WebSocket
Autoreconnect front to back
2020-01-25 14:39:50 +01:00
death-claw 0c0fdf9d78 v2.10.6 2020-01-19 13:01:03 +01:00
death-claw e17afa0ef3 Limit cache size 2020-01-19 12:59:47 +01:00
death-claw 3d3eb6aa36 v2.10.5 2020-01-18 22:28:27 +01:00
death-claw e22132255e Bug fixes 2020-01-18 22:24:53 +01:00
death-claw ef041526ec v2.10.4 2020-01-18 13:27:47 +01:00
death-claw 704311dd9a Zero value for total max download will disable the limit
Add system notification
Bug fixes
2020-01-18 13:21:46 +01:00
death-claw 9617a6f20f v2.10.3 2020-01-16 21:39:59 +01:00
death-claw bc17442961 Some bug fixes
Enhance download queue logic
Clear cache button
Fix the partial status
2020-01-16 21:34:09 +01:00
death-claw 574d6818c1 Refactor PostImgHost 2020-01-14 21:03:35 +01:00
death-clawandGitHub 5c416055ee Merge pull request #3 from life-claw/postimg
Added Support for postimg.cc
2020-01-14 19:01:41 +01:00
death-clawandGitHub fa30458dc6 Merge pull request #1 from life-claw/imagevenue
Add support for imagevenue.com
2020-01-14 18:47:47 +01:00
death-clawandGitHub 638af93225 Merge pull request #2 from life-claw/urlSpaces
Sanitize URLs that contain spaces
2020-01-14 18:14:47 +01:00
life-claw 2c6ca741c7 Added PostImgHost 2020-01-07 16:50:11 -06:00
life-claw d7f2df2540 Sanitize URLs with that contain spaces 2020-01-07 16:18:45 -06:00
life-claw b85ade1dba Added ImageVenueHost 2020-01-07 16:04:04 -06:00
death-claw 1c61a0b34e v2.10.2 2019-12-29 11:35:59 +01:00
death-claw 46b9449969 Upgrade electron to v7
Update the retry policy
2019-12-29 11:33:03 +01:00
death-claw cc1bbd1231 v2.10.1 2019-12-27 23:05:10 +01:00
death-claw b29cba2911 Bug fix 2019-12-27 22:58:33 +01:00
death-claw 13bde3aa2c 2.10.0 2019-12-27 21:27:27 +01:00
death-claw 35b4f5fc97 Add support for imgspice
Fix spring dependency bug
2019-12-27 21:25:33 +01:00
death-claw 1b0482b7d4 v2.9.1 2019-12-27 20:10:31 +01:00
death-claw c9ff6eeab6 Fix build issue 2019-12-27 20:09:16 +01:00
death-claw 5ac57b743d v2.9.0 2019-12-27 20:07:19 +01:00
death-claw f7b219701f Allow many hosts to run at the same time
Add title to gallery
Add global concurrent downloads
Decrease timeout for fast fail
Add counter badge in link collector
2019-12-27 20:03:35 +01:00
death-claw d009aa3f8c v2.8.1 2019-12-24 20:21:29 +01:00
death-claw 078d49f883 v2.8.1 2019-12-24 20:18:22 +01:00
death-claw f1bf517ba5 Fix issue with path creation 2019-12-24 20:15:03 +01:00
death-claw 6269ed6f6e v2.8.0 2019-12-24 19:47:17 +01:00
death-claw 43b8dedbb1 Use pointer cursor on gallery items 2019-12-24 19:42:45 +01:00
death-claw 978be7bd49 Add view photos settings
Some improvements
2019-12-23 22:46:46 +01:00
death-claw 1b7f3aef62 Add missing css and script includes 2019-12-23 19:36:07 +01:00
death-claw 7d24ee4867 Use photoswipe for image gallery 2019-12-23 00:02:13 +01:00
death-claw 269a3475b0 Generate thumbnails on the fly and cache them 2019-12-21 19:17:51 +01:00
death-claw 18e2bc6afc Add Photo Viewer 2019-12-21 17:23:22 +01:00
death-claw c6ef54e115 v2.7.1 2019-12-21 11:24:03 +01:00
death-claw 441b2831fc Add version to title 2019-12-21 11:16:53 +01:00
death-claw 3cf4d60eca v2.7.0 2019-12-21 09:59:17 +01:00
death-claw d228c89a1c Add support for pixxxels
Add missing icon
2019-12-21 09:56:57 +01:00
death-claw f3b592e81b v2.6.0 2019-12-21 08:39:04 +01:00
death-claw a5ff659fb4 Better quality app icon
Implement context menu on download tab
Add stop remove and start button to context menu
Link will not be removed automatically from link collector tab
Some UI fixes
2019-12-21 08:34:56 +01:00
death-claw ab80e2f3b3 v2.5.1 2019-12-09 20:26:52 +01:00
death-claw 2ef082f87e Add hosts column in Link collector grab menu
Disable download button when nothing selected in Link collector grab menu
2019-12-09 19:56:26 +01:00
death-claw a1b97e76a7 v2.5.0 2019-12-08 20:13:58 +01:00
death-claw 185a078b4d Missing change 2019-12-08 20:08:42 +01:00
death-claw 28a4c944ee Add support for multiple links at a time
Add the Link collector feature
Fix bug with previews
2019-12-08 20:07:13 +01:00
death-claw 23d3553d99 Include startup icon
Fix UI layout
2019-12-06 23:09:57 +01:00
death-claw fb09e4c46b v2.4.1 2019-11-26 20:03:10 +01:00
death-claw 4feb6f5fe1 Increase attempts for establishing ws connection between front and back end
Change default download folder to Home folder (installer), App folder (portable)
2019-11-26 20:01:08 +01:00
death-claw df1b9efbbb v2.4.0 2019-11-23 14:50:16 +01:00
death-claw 553924fa1d Pimpandhost Host support 2019-11-23 14:48:05 +01:00
death-claw a90dbd568f v2.3.2 2019-11-23 12:15:42 +01:00
death-claw ca7ca1ce8e Fix a bug with the scan component 2019-11-23 12:14:07 +01:00
death-claw b3ab239a1e v2.3.1 2019-11-22 20:07:41 +01:00
death-claw 6e212c791c Fix connexion reset issue
Fix Post Status when autostart option is selected
Add stop all button
2019-11-22 20:05:14 +01:00
death-claw c6067ef147 v2.3.0 2019-11-21 21:17:25 +01:00
death-claw 8333f0c34e Fix Restoring post status on App startup
Use local sockjs and material icons font
2019-11-21 21:16:24 +01:00
death-claw 75becd8bb4 UI enhancements and overhaul
Stability fixes with HTTP connections
MacOS official support
2019-11-21 20:54:44 +01:00
death-claw 667376301a v2.2.5 2019-11-09 13:26:54 +01:00
death-claw 128e6b68a3 More robust parsing
Fix pixhost and imagezilla to exclude thumbs
2019-11-09 13:16:09 +01:00
death-claw a7c2844ab4 Change Retry duration 2019-11-09 10:13:12 +01:00
death-claw bd0b7d274b Fix macOs paths 2019-10-06 13:33:29 +01:00
death-claw bc57fad52b Remove failing rule 2019-10-06 12:00:01 +01:00
death-claw b81ef53cb9 Add support for macOs 2019-10-06 11:22:48 +01:00
death-claw c182bf0a08 v2.2.4 2019-10-05 18:06:40 +01:00
death-claw 07802b4b13 Add AppImage target 2019-10-05 18:03:32 +01:00
death-claw 50c4d947ed v2.2.3 2019-10-01 23:28:12 +01:00
death-claw b9b20e86dd Strip libjvm.so for debian 2019-10-01 23:27:01 +01:00
death-claw 64b93ce7e1 v2.2.2 2019-10-01 22:59:39 +01:00
death-claw 0bde4399e3 Update product name 2019-10-01 22:57:13 +01:00
death-claw 9b7818c5a7 v2.2.1 2019-10-01 22:44:19 +01:00
death-claw 1f2dcff353 Update dependency list for deb package 2019-10-01 22:42:28 +01:00
death-claw 4d82622fdb v2.2.0 2019-10-01 21:26:27 +01:00
death-claw e7393caf01 Fix 'Start' on pending items
Upgrade to Java 11
Embedd JRE within the app
2019-10-01 21:24:02 +01:00
death-claw 8e5cb958ea v2.1.0 2019-09-30 21:12:56 +01:00
death-claw bc2cf02ec6 Add a create a subfolder per thread option 2019-09-30 21:06:40 +01:00
death-claw 7a8ecb306f v2.0.4 2019-09-27 20:52:25 +01:00
death-claw 76c3901afa Fix some issues with ImxHost 2019-09-27 20:50:51 +01:00
death-claw f16b2227a1 v2.0.3 2019-09-23 17:38:36 +01:00
death-claw d8d7ca5408 Update deb dependencies 2019-09-23 17:36:10 +01:00
death-claw 28c6b71e99 v2.0.2 2019-09-23 16:46:37 +01:00
death-claw c40d3e2377 Add portable mode 2019-09-23 16:43:52 +01:00
death-claw 2217fea0ad v2.0.1 2019-09-22 21:55:30 +01:00
death-claw 94746441b5 Update app name 2019-09-22 21:48:35 +01:00
death-claw c3e861fa8b v2.0.0 2019-09-22 20:58:24 +01:00
death-claw ae1d644dba Update icons 2019-09-22 20:41:32 +01:00
death-claw 1fcb7e4e42 Fix a bug in data file 2019-09-22 20:29:34 +01:00
death-claw 646cbba21c Update package.json 2019-09-22 20:21:05 +01:00
death-claw eef4d13235 Change data and log paths 2019-09-22 20:15:03 +01:00
death-claw 9d1fe4c6b6 Electron app changes 2019-09-22 18:41:27 +01:00
death-claw 610d08f4ca Add sub folder option
Add clear finished option
UI enhancements
Bug fixes
2019-09-22 16:23:52 +01:00
death-claw 7e52d0d6de v1.6.5 2019-09-18 23:20:58 +01:00
death-claw 4f0c5925ad electron update 2019-09-18 23:19:01 +01:00
308 changed files with 19187 additions and 13465 deletions
+3 -1
View File
@@ -2,4 +2,6 @@
/**/*/node_modules
/**/*/target
/**/*/build-dir
.idea
/**/*/java-runtime
.idea
.vripper
+318 -1
View File
@@ -1,5 +1,322 @@
# Changelog
## [3.2.1] - 2021-01-12
### Added
- Add settings for connection timeout
- Add settings for maximum attempts
### Changed
- Enhance posts rename logic
- Dependencies update
- Switch reactive from rx-java to reactor
## [3.2.0] - 2020-11-16
### Changed
- Fix build issue
## [3.1.0] - 2020-11-16
### Changed
- UI revamp
- Fix bugs
## [3.0.11] - 2020-10-31
### Added
- Support for vipr.im
## [3.0.10] - 2020-10-04
### Changed
- Fix bug on settings update
## [3.0.9] - 2020-09-27
### Changed
- Revert to appending post id to download folders
- Fix a bug that lead to adding duplicate posts
- Remove old unused settings
## [3.0.8] - 2020-08-22
### Changed
- Bug fix which breaks concurrent downloads
## [3.0.7] - 2020-08-22
### Changed
- Fix MacOs build
## [3.0.6] - 2020-08-18
### Changed
- Fix TurboImageHost
## [3.0.5] - 2020-08-17
### Changed
- Update packages
- Host class refactoring
## [3.0.4] - 2020-08-16
### Changed
- Improve rename UI
## [3.0.3] - 2020-08-16
### Changed
- Fix bugs with electron app
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
- Introducing a proper database for persistence
## [2.12.1] - 2020-05-22
### Changed
- Fix alternative name bug
## [2.12.0] - 2020-05-21
### Changed
- Use new user hash from the api
## [2.11.7] - 2020-05-08
### Changed
- Fix bug with paths creation
- Move Open Link button
- Rename rename all title
## [2.11.6] - 2020-05-04
### Changed
- PNG filenames are now correctly named
- Fix issue with special characters in post titles
## [2.11.5] - 2020-05-03
### Changed
- Reduce rows buffer size
## [2.11.4] - 2020-05-03
### Changed
- Code cleanup
- Dependencies update
## [2.11.3] - 2020-04-30
### Changed
- Source code restructure
- UI fixes and enhancements
### Added
- Add menu entry to rename a gallery to first alternative title
- Add menu on toolbar for massive rename to first alternative title
## [2.11.2] - 2020-04-26
### Changed
- Bug fixes
## [2.11.1] - 2020-04-25
### Changed
- Code source maintenance
### Added
- Added option to rename gallery
## [2.11.0] - 2020-03-29
### Changed
- Upgrade Electron to v8
- Upgrade Angular to v9
- App settings are now portable in a json file
- General code source maintenance
### Added
- Added Support for pixroute.com
- Added grab post title from thread option
## [2.10.8] - 2020-01-25
### Changed
- Fix app url
## [2.10.7] - 2020-01-25
### Changed
- Drop SockJs and switch to WebSocket
- Autoreconnect front to back
## [2.10.6] - 2020-01-19
### Changed
- Limit cache size
## [2.10.5] - 2020-01-18
### Changed
- Bug fixes
## [2.10.4] - 2020-01-18
### Changed
- Zero value for total max download will disable the limit
- Add system notification
- Bug fixes
## [2.10.3] - 2020-01-16
### Changed
- Some bug fixes
- Enhance download queue logic
- Clear cache button
- Fix the partial status
- Sanitize URLs with spaces
### Added
- Support for Postimg host
- Support for Imagevenue host
## [2.10.2] - 2019-12-29
### Changed
- Upgrade electron to v7
- Bug fix
## [2.10.1] - 2019-12-27
### Changed
- Bug fix
## [2.10.0] - 2019-12-27
### Changed
- Fix spring dependency bug
### Added
- Add support for imgspice
## [2.9.0] - 2019-12-27
### Changed
- Fix build issue
## [2.9.0] - 2019-12-27
### Changed
- Decrease timeout for fast fail
### Added
- Allow many hosts to run at the same time
- Add title to gallery
- Add global concurrent downloads
- Add counter badge in link collector
## [2.8.1] - 2019-12-24
### Changed
- Fix issue with path creation
## [2.8.0] - 2019-12-24
### Added
- Photo gallery feature
## [2.7.1] - 2019-12-21
### Added
- Add version to title bar
## [2.7.0] - 2019-12-21
### Changed
- Add missing icon
### Added
- Add support for pixxxels
## [2.6.0] - 2019-12-21
### Changed
- Disable download button when nothing selected in Link collector grab menu
- Better quality app icon
- Implement context menu on download tab
- Add stop remove and start button to context menu
- Link will not be removed automatically from link collector tab
- Some UI fixes
## [2.5.1] - 2019-12-09
### Changed
- Disable download button when nothing selected in Link collector grab menu
### Added
- Add hosts column in Link collector grab menu
## [2.5.0] - 2019-12-08
### Changed
- Change loading screen
- UI Fixes
- Fix bug with previews
### Added
- Add link collector feature
- Add support for multiple links at a time
## [2.4.1] - 2019-11-26
### Changed
- Increase attempts for establishing ws connection between front and back end
- Change default download folder to Home folder (installer), App folder (portable)
## [2.4.0] - 2019-11-23
### Changed
- Pimpandhost Host support
## [2.3.2] - 2019-11-23
### Changed
- Fix a bug with the scan component
## [2.3.1] - 2019-11-22
### Changed
- Fix connexion reset issue
- Fix Post Status when autostart option is selected
- Add stop all button
## [2.3.0] - 2019-11-21
### Changed
- UI enhancements and overhaul
- Stability fixes with HTTP connections
- MacOS official support
- Use local sockjs and material icons font
## [2.2.5] - 2019-11-09
### Changed
- Exclude thumbs download for imagezilla and pixhost
- Change retry on fail logic (may help avoid having download errors)
## [2.2.4] - 2019-10-05
### Added
- AppImage target for linux
## [2.2.3] - 2019-10-01
### Changed
- Strip libjvm.so for debian
## [2.2.2] - 2019-10-01
### Changed
- Update product name
## [2.2.1] - 2019-10-01
### Changed
- Update dependency list for deb package
## [2.2.0] - 2019-10-01
### Added
- Embedd JRE within the app
### Changed
- Fix 'Start' on pending items
- Upgrade to Java 11
## [2.1.0] - 2019-09-30
### Added
- Add a create a subfolder per thread option
## [2.0.4] - 2019-09-27
### Changed
- Fix some issues with ImxHost
## [2.0.3] - 2019-09-23
### Changed
- Update deb dependencies
## [2.0.2] - 2019-09-23
### Changed
- Add portable mode
## [2.0.1] - 2019-09-22
### Changed
- Change app name
## [2.0.0] - 2019-09-22
### Added
- Add sub folder option
- Add clear finished option
### Changed
- UI enhancements
- Bug fixes
- Change data and log paths
## [1.6.5] - 2019-09-18
### Changed
- electron update
## [1.6.4] - 2019-09-18
### Changed
- Change menu item name
@@ -126,4 +443,4 @@
- Add real time communication for progress updates via websocket (SockJS)
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
+6 -8
View File
@@ -1,26 +1,24 @@
# Viper Ripper!
This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website.
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
## How to build
You need a recent version of maven 3.6.1+.
The application support 2 build modes:
- Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
- Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
To build the server app:
mvn clean install
mvn clean install -DskipTests
To build the desktop app:
mvn clean install -Pelectron
mvn clean install -Pelectron -DskipTests
Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
*Partially cross platform:
Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.
[![Donate with PayPal](https://raw.githubusercontent.com/stefan-niedermann/paypal-donate-button/master/paypal-donate-button.png)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=SC7M5LNWZ528Q)
+3 -3
View File
@@ -4,12 +4,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>1.6.4</version>
<version>3.2.1</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.1.RELEASE</version>
<version>2.4.1</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<modules>
@@ -17,4 +17,4 @@
<module>vripper-ui</module>
<module>vripper-electron</module>
</modules>
</project>
</project>
Binary file not shown.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 303 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 592 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

+142 -111
View File
@@ -1,13 +1,16 @@
const { app, BrowserWindow } = require("electron");
require('v8-compile-cache');
const {app, BrowserWindow} = require("electron");
const path = require("path");
const url = require("url");
const getPort = require("get-port");
const { spawn } = require("child_process");
const { ipcMain } = require("electron");
const commandExists = require("command-exists").sync;
const { dialog } = require("electron");
const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const appDir = process.env.APPDIR;
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
const appImagePath = process.env.APPIMAGE;
let win;
let vripperServer;
@@ -19,122 +22,150 @@ let terminationInteval;
let terminated = false;
process.on("uncaughtException", err => {
dialog.showErrorBox(err.message, err.stack);
process.exit(1);
dialog.showErrorBox(err.message, err.stack);
process.exit(1);
});
if (!commandExists("java")) {
dialog.showErrorBox(
"Java command is missing",
"Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
);
process.exit(1);
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId("tn.mnlr.vripper");
}
let icon;
if (process.platform === "win32") {
icon = __dirname + '/icon.ico';
} else {
icon = __dirname + '/icon.png';
}
win = new BrowserWindow({
width: 1024,
height: 800,
minWidth: 800,
minHeight: 600,
webPreferences: {
nodeIntegration: true,
enableRemoteModule: true
},
icon: icon
});
win.removeMenu();
win.setMenu(null);
win.setMenuBarVisibility(false);
win.loadURL(
url.format({
pathname: path.join(__dirname, `vripper-ui/index.html`),
protocol: "file:",
slashes: true
})
);
// win.webContents.openDevTools();
win.on("closed", () => {
win = null;
});
}
function createWindow() {
let icon;
if(process.platform === "win32") {
icon = path.join(__dirname, `icon.ico`);
} else {
icon = path.join(__dirname, `icon.icns`);
}
win = new BrowserWindow({
width: 1024,
height: 800,
minWidth: 640,
minHeight: 480,
frame: false,
webPreferences: {
nodeIntegration: true
},
icon: icon
});
win.removeMenu();
win.loadURL(
url.format({
pathname: path.join(__dirname, `vripper-ui/index.html`),
protocol: "file:",
slashes: true
})
);
// win.webContents.openDevTools();
win.on("closed", () => {
win = null;
});
shutdownServer = () => {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: {'content-type': 'application/json'},
}).then((response) => {
terminationInteval = setInterval(() => {
terminationAttemps++;
if (terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if (terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
}).catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
getPort().then(port => {
serverPort = port;
ipcMain.on("get-port", event => {
event.reply("port", port);
});
vripperServer = spawn("java", [
"-Xms256m",
"-Xmx1024m",
"-Dvripper.server.port=" + port,
"-jar",
appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
path.join(app.getPath('exe'), "../bin/vripper-server.jar")
], {
stdio: 'ignore'
});
vripperServer.on('exit', (code, signal) => {
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
terminated = true;
app.quit();
});
});
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
app.quit();
app.quit();
} else {
app.on("second-instance", (event, commandLine, workingDirectory) => {
if (win) {
if (win.isMinimized()) win.restore();
win.focus();
}
});
getPort().then(port => {
serverPort = port;
ipcMain.on("get-port", event => {
event.reply("port", port);
});
app.on("ready", createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: { 'content-type': 'application/json' },
}).then((response) => {
terminationInteval= setInterval(() => {
terminationAttemps++;
if(terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
const appPath = path.join(app.getAppPath(), '../../');
let javaBinPath, jarPath, baseDir;
if (appImageDir !== undefined && process.platform === 'linux') {
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
baseDir = path.join(appImagePath, '..');
} else if (process.platform === 'darwin') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = app.getPath('appData');
} else if (process.platform === 'win32') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = appPath;
} else {
console.error(`Unknown platform ${process.platform}`);
app.quit();
} else if(terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
})
.catch((error) => {
// Terminate immediately
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
});
}
app.on("activate", () => {
if (win === null) {
createWindow();
}
});
vripperServer = spawn(javaBinPath, [
"-Xms256m",
"-Dvripper.server.port=" + port,
"-Dbase.dir=" + baseDir,
"-jar",
jarPath
], {
stdio: 'ignore'
});
vripperServer.on('exit', (code, signal) => {
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
terminated = true;
app.quit();
});
});
app.on("second-instance", (event, commandLine, workingDirectory) => {
if (win) {
if (win.isMinimized()) win.restore();
win.focus();
}
});
app.on("ready", createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
shutdownServer();
}
});
app.on("will-quit", () => {
if (process.platform === "darwin") {
shutdownServer();
}
});
app.on("activate", () => {
if (win === null) {
createWindow();
}
});
}
+1176 -1959
View File
File diff suppressed because it is too large Load Diff
+35 -16
View File
@@ -1,12 +1,14 @@
{
"name": "vripper-electron",
"version": "1.6.4",
"description": "",
"version": "3.2.1",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
"homepage": "https://github.com/death-claw/vripper-project",
"license": "ISC",
"build": {
"appId": "tn.mnlr.vripper",
"productName": "VRipper",
"files": [
"**/*",
"!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -29,33 +31,50 @@
{
"from": "../vripper-server/target/vripper-server-${version}-electron.jar",
"to": "bin/vripper-server.jar"
},
{
"from": "java-runtime",
"to": "java-runtime"
}
],
"win": {
"target": "nsis"
"icon": "icon.ico",
"target": [
"dir"
]
},
"linux": {
"synopsis": "vipergirls.to ripper",
"category": "Utility",
"packageCategory": "Utility",
"icon": "icons",
"target": [
"zip"
"AppImage"
]
},
"mac": {
"category": "public.app-category.utilities",
"target": [
"dmg"
],
"icon": "icon.icns"
}
},
"scripts": {
"start": "electron .",
"dist": "node pre-build.js && build"
"dist": "node pre-build.js && electron-builder"
},
"devDependencies": {
"electron": "^6.0.4",
"electron-builder": "^20.44.4"
"electron": "11.1.1",
"electron-builder": "22.9.1"
},
"dependencies": {
"axios": "^0.19.0",
"cheerio": "^1.0.0-rc.3",
"command-exists": "^1.2.8",
"copy-dir": "^1.1.0",
"custom-electron-titlebar": "^3.0.10",
"electron-context-menu": "^0.13.0",
"get-port": "^5.0.0",
"rimraf": "^2.6.3"
"axios": "0.21.1",
"cheerio": "1.0.0-rc.3",
"copy-dir": "1.3.0",
"electron-context-menu": "2.1.0",
"get-port": "5.1.1",
"rimraf": "3.0.2",
"v8-compile-cache": "2.1.1"
}
}
+71 -71
View File
@@ -1,77 +1,77 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>1.6.4</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<classifier>${buildClassifier}</classifier>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.2.1</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.3</version>
<configuration>
<nodeVersion>v12.2.0</nodeVersion>
<npmVersion>6.9.0</npmVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<arguments>run dist</arguments>
<classifier>${buildClassifier}</classifier>
</configuration>
<phase>generate-resources</phase>
</execution>
</executions>
</plugin>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.3</version>
<configuration>
<nodeVersion>v12.2.0</nodeVersion>
<npmVersion>6.9.0</npmVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run dist</arguments>
</configuration>
<phase>generate-resources</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
+8 -1
View File
@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
const fs = require('fs');
const rimraf = require("rimraf");
const copydir = require("copy-dir");
const {execSync} = require("child_process");
rimraf.sync('./build/vripper-ui');
copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
fs.writeFileSync('./build/vripper-ui/index.html', $.html());
console.log('Building runtime environment');
rimraf.sync('java-runtime');
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
if (process.platform === 'linux') {
console.log('Stripping libjvm.so');
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
}
+1 -13
View File
@@ -1,15 +1,3 @@
console.log("renderer intialized");
const customTitlebar = require("custom-electron-titlebar");
const electron = require("electron").remote;
const contextMenu = require("electron-context-menu");
contextMenu({});
const menu = new electron.Menu();
const titlebar = new customTitlebar.Titlebar({
backgroundColor: customTitlebar.Color.fromHex("#000000"),
titleHorizontalAlignment: 'left',
icon: 'favicon.ico',
shadow: true,
menu: menu
});
titlebar.updateTitle("Viper Ripper");
contextMenu({});
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
+23 -5
View File
@@ -5,14 +5,14 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>1.6.4</version>
<version>3.2.1</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
<description>vripper-server</description>
<properties>
<java.version>1.8</java.version>
<java.version>11</java.version>
</properties>
<dependencies>
@@ -32,6 +32,20 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.5.1</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -58,15 +72,19 @@
<version>2.22</version>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
<artifactId>rxjava</artifactId>
<version>2.2.6</version>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>net.jodah</groupId>
<artifactId>failsafe</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>28.1-jre</version>
</dependency>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-ui</artifactId>
@@ -0,0 +1,36 @@
package tn.mnlr.vripper;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.jpa.repositories.IRepository;
import tn.mnlr.vripper.services.DataService;
import java.util.Set;
@Component
public class EventListenerBean {
@Getter
private static boolean init = false;
private final DataService dataService;
private final Set<IRepository> repositorySet;
@Autowired
public EventListenerBean(DataService dataService, Set<IRepository> repositorySet) {
this.dataService = dataService;
this.repositorySet = repositorySet;
}
@EventListener
public void onApplicationEvent(ContextRefreshedEvent event) {
repositorySet.forEach(IRepository::init);
dataService.setDownloadingToStopped();
init = true;
}
}
@@ -1,7 +1,6 @@
package tn.mnlr.vripper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@@ -11,30 +10,24 @@ import org.springframework.stereotype.Component;
import java.util.Map;
@Component
@Slf4j
public class SpringContext implements ApplicationContextAware {
private static final Logger logger = LoggerFactory.getLogger(SpringContext.class);
private static ConfigurableApplicationContext context;
/**
* Returns the Spring managed bean instance of the given class type (if it exists).
* Returns null otherwise.
*
* @param beanClass
* @return
*/
public static <T extends Object> T getBean(Class<T> beanClass) {
public static <T> T getBean(Class<T> beanClass) {
return context.getBean(beanClass);
}
public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
return context.getBeansOfType(beanClass);
}
public static void close() {
logger.info("Application terminating...");
context.close();
log.info("Application terminating...");
if (context != null) {
context.close();
}
}
@@ -1,87 +1,20 @@
package tn.mnlr.vripper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.PersistenceService;
import tn.mnlr.vripper.services.PostParser;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import java.io.File;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@SpringBootApplication
@Slf4j
public class VripperApplication {
private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
public static final String dataPath = System.getProperty("vripper.datapath", ".") + File.separator + "data.json";
public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
public static void main(String[] args) {
try {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
SpringApplication.run(VripperApplication.class, args);
} catch (Exception e) {
logger.error("Failed to run the application", e);
}
}
@Component
public class AppCommandRunner implements CommandLineRunner {
@Autowired
private VipergirlsAuthService authService;
@Autowired
private PersistenceService persistenceService;
@Autowired
private AppSettingsService appSettingsService;
@Override
public void run(String... args) {
persistenceService.restore();
appSettingsService.restore();
registerShutdownHook();
try {
authService.authenticate();
} catch (VripperException e) {
logger.error("Cannot authenticate user with ViperGirls", e);
}
}
private void registerShutdownHook() {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
}
}
@Component
public class TestRunner implements CommandLineRunner {
@Autowired
private PostParser postParser;
@Override
public void run(String... args) throws Exception {
// long start = System.currentTimeMillis();
// PostParser.VRThreadParser vrThreadParser = postParser.new VRThreadParser("600408");
// vrThreadParser.getPostPublishProcessor().subscribe(e -> System.out.println("new Post: " + e.getPostId()));
// vrThreadParser.parse();
// System.out.println("Total time: " + (System.currentTimeMillis() - start));
log.error("Failed to run the application", e);
}
}
}
@@ -0,0 +1,277 @@
package tn.mnlr.vripper.download;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.function.CheckedRunnable;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.*;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
@Slf4j
public class DownloadJob implements CheckedRunnable {
private static final Object LOCK = new Object();
private static final int READ_BUFFER_SIZE = 8192;
private final DataService dataService;
private final PathService pathService;
private final ConnectionService cm;
private final VGAuthService authService;
private final DownloadSpeedService downloadSpeedService;
private final SettingsService settingsService;
private final HttpClientContext context;
@Getter
private final Image image;
@Getter
private final Post post;
private boolean stopped = false;
@Getter
private boolean finished = false;
DownloadJob(Post post, Image image) {
this.image = image;
this.post = post;
dataService = SpringContext.getBean(DataService.class);
pathService = SpringContext.getBean(PathService.class);
cm = SpringContext.getBean(ConnectionService.class);
authService = SpringContext.getBean(VGAuthService.class);
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
settingsService = SpringContext.getBean(SettingsService.class);
context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
context.setAttribute(ContextAttributes.OPEN_CONNECTION.toString(), Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
}
public void download(final Post post, final Image image) throws DownloadException {
try {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
// The post may be updated and the download directory might be set by another thread
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
if (updatedPost.getDownloadDirectory() == null) {
pathService.createDefaultPostFolder(updatedPost);
}
if (settingsService.getSettings().getLeaveThanksOnStart()) {
authService.leaveThanks(updatedPost);
}
}
if (stopped) {
return;
}
/*
* HOST SPECIFIC
*/
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
/*
* END HOST SPECIFIC
*/
if (stopped) {
return;
}
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
log.debug(String.format("Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
HttpClient client = cm.getClient().build();
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
httpGet.addHeader("Referer", image.getUrl());
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (stopped) {
return;
}
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
if (stopped) {
return;
}
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
if (stopped) {
return;
}
}
checkImageTypeAndRename(dataService.findById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
}
} catch (Exception e) {
if (stopped) {
return;
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
image.setStatus(Status.COMPLETE);
} else if (stopped) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
finished = true;
}
}
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
imageName += ".jpg";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_jpg")) {
toReplace = "_jpg";
} else if (imageNameLC.endsWith("_jpeg")) {
toReplace = "_jpeg";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
}
}
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_png")) {
imageName += ".png";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_png")) {
toReplace = "_png";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
}
}
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
pathService.getDirectoryAccess().lock();
File downloadDestinationFolder = pathService.calcDownloadDirectory(post);
File outImage = new File(downloadDestinationFolder, (settingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
} finally {
pathService.getDirectoryAccess().unlock();
try {
Files.delete(outputFile.toPath());
} catch (IOException e) {
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
}
}
}
@Override
public void run() throws Exception {
if (stopped) {
finished = true;
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
download(post, image);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return Objects.equals(image, that.image) &&
Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image, post);
}
public void stop() {
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
this.stopped = true;
}
public enum ContextAttributes {
OPEN_CONNECTION("OPEN_CONNECTION");
private final String value;
ContextAttributes(String value) {
this.value = value;
}
@Override
public String toString() {
return value;
}
}
}
@@ -0,0 +1,40 @@
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.DataService;
@Slf4j
public class DownloadRunnable implements Runnable {
private final DownloadService downloadService;
private final DataService dataService;
private final ConnectionService connectionService;
private final DownloadJob downloadJob;
public DownloadRunnable(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
connectionService = SpringContext.getBean(ConnectionService.class);
this.downloadJob = downloadJob;
}
@Override
public void run() {
Failsafe.with(connectionService.getRetryPolicy())
.onFailure(e -> {
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
})
.onComplete(e -> {
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
downloadService.afterJobFinish(downloadJob);
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
}).run(downloadJob);
}
}
@@ -0,0 +1,223 @@
package tn.mnlr.vripper.download;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.SettingsService;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@Slf4j
public class DownloadService {
private static final List<Status> FINISHED = Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
private final int MAX_POOL_SIZE = 12;
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
private final PendingQueue pendingQueue;
private final SettingsService settings;
private final DataService dataService;
private final PostService postService;
private boolean pauseQ = false;
private Thread executionThread;
private Thread pollThread;
@Autowired
public DownloadService(PendingQueue pendingQueue, SettingsService settings, DataService dataService, PostService postService) {
this.pendingQueue = pendingQueue;
this.settings = settings;
this.dataService = dataService;
this.postService = postService;
}
@PostConstruct
private void init() {
executionThread = new Thread(this::start, "Executor thread");
pollThread = new Thread(this::poll, "Polling thread");
pollThread.start();
executionThread.start();
}
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
executionThread.interrupt();
pollThread.interrupt();
executor.shutdown();
dataService.findAllPosts().forEach(p -> {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
executor.awaitTermination(5, TimeUnit.SECONDS);
}
private void stopRunning(@NonNull String postId) {
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = executing.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::stop);
} else {
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
}
}
public void restartAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::restart);
} else {
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
}
}
private void restart(@NonNull String postId) {
if (isPending(postId)) {
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
return;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
post.setStatus(Status.PENDING);
dataService.updatePostStatus(post.getStatus(), post.getId());
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
try {
pendingQueue.put(post, image);
} catch (InterruptedException e) {
log.warn("Thread was interrupted", e);
Thread.currentThread().interrupt();
}
}
}
private boolean isPending(String postId) {
return pendingQueue.isPending(postId);
}
private void stop(String postId) {
try {
pauseQ = true;
final Post post = dataService.findPostByPostId(postId).orElseThrow();
if (post == null) {
return;
}
if (FINISHED.contains(post.getStatus())) {
return;
}
pendingQueue.stop(post);
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
postService.stopFetchingMetadata(post);
} finally {
pauseQ = false;
}
}
private boolean canRun(Host host) {
boolean canRun;
AtomicInteger count = threadCount.get(host);
if (count == null) {
threadCount.put(host, new AtomicInteger(0));
}
canRun = threadCount.get(host).get() < settings.getSettings().getMaxThreads() && (settings.getSettings().getMaxTotalThreads() == 0 ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE : threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getSettings().getMaxTotalThreads());
if (canRun && !pauseQ) {
threadCount.get(host).incrementAndGet();
return true;
}
return false;
}
private void poll() {
while (!Thread.interrupted()) {
try {
List<DownloadJob> peek = pendingQueue.peek();
for (DownloadJob downloadJob : peek) {
if (canRun(downloadJob.getImage().getHost())) {
executionQueue.offer(downloadJob);
pendingQueue.remove(downloadJob);
}
}
synchronized (threadCount) {
threadCount.wait(2_000);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
private void start() {
while (!Thread.interrupted()) {
try {
push(executionQueue.take());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
} catch (Exception e) {
log.error("Execution Service failed", e);
break;
}
}
}
private void push(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new DownloadRunnable(downloadJob));
executing.add(downloadJob);
}
public synchronized void afterJobFinish(DownloadJob downloadJob) {
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
if (count == 0) {
dataService.finishPost(downloadJob.getPost());
}
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
executing.remove(downloadJob);
synchronized (threadCount) {
threadCount.notify();
}
}
public int runningCount() {
return executing.size();
}
}
@@ -0,0 +1,115 @@
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.SettingsService;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
@Service
@Slf4j
public class PendingQueue {
private final DataService dataService;
private final SettingsService settingsService;
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
@Autowired
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
this.dataService = dataService;
this.settingsService = settingsService;
this.hosts = hosts;
}
@PostConstruct
private void init() {
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
}
public void put(Post post, Image image) throws InterruptedException {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob = new DownloadJob(post, image);
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
checkKey(post.getPostId());
toBeExecuted.get(post.getPostId()).incrementAndGet();
}
private synchronized void checkKey(String postId) {
if (!toBeExecuted.containsKey(postId)) {
toBeExecuted.put(postId, new AtomicInteger(0));
}
}
public void remove(final DownloadJob downloadJob) {
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
}
public List<DownloadJob> peek() {
List<DownloadJob> downloadJobs = new ArrayList<>();
if (hosts.size() == 0) {
return downloadJobs;
}
for (Host host : hosts) {
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
if (downloadJob != null) {
downloadJobs.add(downloadJob);
}
}
}
return downloadJobs;
}
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
for (Image image : images) {
put(post, image);
}
}
public int size() {
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
}
public void stop(Post post) {
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
entry.getValue().removeIf(predicate);
decrement(post.getPostId());
}
}
public boolean isPending(String postId) {
return toBeExecuted.containsKey(postId);
}
public synchronized int decrement(String postId) {
AtomicInteger counter = toBeExecuted.get(postId);
if (counter == null) {
return 0;
}
int count = counter.decrementAndGet();
if (count == 0) {
toBeExecuted.remove(postId);
}
return count;
}
}
@@ -1,134 +0,0 @@
package tn.mnlr.vripper.entities;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
@Getter
@NoArgsConstructor
public class Image {
private static final Logger logger = LoggerFactory.getLogger(Image.class);
private AppStateService appStateService;
private Disposable subscription;
private final String type = "img";
private String postId;
private String postName;
private Host host;
private String url;
private int index;
private AtomicLong current = new AtomicLong(0);
private Status status;
private BehaviorProcessor<Image> imageStateProcessor;
@Setter
private long total = 0;
public Image(String url, String postId, String postName, Host host, int index) {
this.url = url;
this.postId = postId;
this.postName = postName;
this.host = host;
this.index = index;
status = Status.PENDING;
imageStateProcessor = BehaviorProcessor.create();
}
public void setAppStateService(AppStateService appStateService) {
this.appStateService = appStateService;
appStateService.getCurrentImages().put(this.url, this);
appStateService.getLiveImageUpdates().onNext(this);
}
public void setStatus(Status status) {
this.status = status;
update();
}
public boolean isCompleted() {
return status.equals(Status.COMPLETE);
}
public void init() {
if (appStateService == null) {
logger.warn("Attempting to init the image whilst the App State is null, unexpected behaviour could occur from this");
return;
}
cleanup();
imageStateProcessor = BehaviorProcessor.create();
subscription = imageStateProcessor
.onBackpressureLatest()
.doOnNext(appStateService::onImageUpdate)
.subscribe();
current.set(0);
status = Status.PENDING;
imageStateProcessor.onNext(this);
}
public void cleanup() {
if (imageStateProcessor != null) {
imageStateProcessor.onComplete();
}
if (subscription != null) {
subscription.dispose();
}
}
public void setCurrent(int current) {
this.current.set(current);
update();
}
public void increase(int read) {
current.addAndGet(read);
update();
}
private void update() {
if(imageStateProcessor == null) {
return;
}
imageStateProcessor.onNext(this);
if (isCompleted()) {
imageStateProcessor.onComplete();
subscription.dispose();
imageStateProcessor = null;
}
}
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return url.equals(image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -1,115 +0,0 @@
package tn.mnlr.vripper.entities;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Getter
@ToString
@NoArgsConstructor
public class Post {
private AppStateService appStateService;
private Status status;
private final String type = "post";
private String postId;
private String threadId;
private String title;
private String postCounter;
private String url;
private List<Image> images;
private Map<String, Object> metadata;
private AtomicInteger done = new AtomicInteger(0);
private int total;
private Set<String> hosts;
private boolean removed = false;
public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId) {
this.title = title;
this.url = url;
this.images = images;
this.metadata = metadata;
this.postId = postId;
this.threadId = threadId;
if (this.images.contains(null)) {
System.out.println("Oops");
}
this.images.stream().map(e -> {
if (e == null) {
System.out.println("Am the cause");
}
return e.getHost();
}).collect(Collectors.toSet());
this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
total = images.size();
status = Status.PENDING;
}
public void setAppStateService(AppStateService appStateService) {
this.appStateService = appStateService;
this.appStateService.getCurrentPosts().put(postId, this);
this.appStateService.getLivePostsState().onNext(this);
}
public void setRemoved(boolean removed) {
this.removed = removed;
updateNotification();
}
public void increase() {
done.incrementAndGet();
updateNotification();
}
public void setStatus(Status status) {
this.status = status;
updateNotification();
}
private void updateNotification() {
if (appStateService != null) {
appStateService.getLivePostsState().onNext(this);
}
}
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, PARTIAL, STOPPED
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return postId.equals(post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -1,12 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.databind.util.StdConverter;
import tn.mnlr.vripper.host.Host;
public class HostToString extends StdConverter<Host, String> {
@Override
public String convert(Host value) {
return value.getClass().getSimpleName();
}
}
@@ -1,32 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
public abstract class ImagePersistanceMixin {
@JsonSerialize(converter = HostToString.class)
@JsonDeserialize(converter = StringToHost.class)
private Host host;
@JsonIgnore
private BehaviorProcessor<Image> imageStateProcessor;
@JsonIgnore
private Disposable subscription;
@JsonIgnore
private String type;
@JsonIgnore
public abstract boolean isCompleted();
@JsonIgnore
private AppStateService appStateService;
}
@@ -1,17 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.annotation.JsonIgnore;
import tn.mnlr.vripper.services.AppStateService;
public abstract class PostPersistanceMixin {
@JsonIgnore
private String type;
@JsonIgnore
private AppStateService appStateService;
@JsonIgnore
private boolean removed;
}
@@ -1,17 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.databind.util.StdConverter;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import java.util.Collection;
public class StringToHost extends StdConverter<String, Host> {
private Collection<Host> hosts = SpringContext.getBeansOfType(Host.class).values();
@Override
public Host convert(String value) {
return hosts.stream().filter(e -> e.getClass().getSimpleName().equals(value)).findAny().orElse(null);
}
}
@@ -1,23 +0,0 @@
package tn.mnlr.vripper.entities.mixin.ui;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
public abstract class ImageUIMixin {
@JsonIgnore
private Host host;
@JsonIgnore
private BehaviorProcessor<Image> imageStateProcessor;
@JsonIgnore
private Disposable subscription;
@JsonIgnore
private AppStateService appStateService;
}
@@ -1,16 +0,0 @@
package tn.mnlr.vripper.entities.mixin.ui;
import com.fasterxml.jackson.annotation.JsonIgnore;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
public abstract class PostUIMixin {
@JsonIgnore
private List<Image> images;
@JsonIgnore
private AppStateService appStateService;
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class ImageUpdateEvent extends ApplicationEvent {
private final Long id;
public ImageUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -0,0 +1,17 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class MetadataUpdateEvent extends ApplicationEvent {
private final Long id;
private final Long postIdRef;
public MetadataUpdateEvent(Object source, Long id, Long postIdRef) {
super(source);
this.id = id;
this.postIdRef = postIdRef;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class PostRemoveEvent extends ApplicationEvent {
private final String postId;
public PostRemoveEvent(Object source, String postId) {
super(source);
this.postId = postId;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class PostUpdateEvent extends ApplicationEvent {
private final Long id;
public PostUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class QueuedRemoveEvent extends ApplicationEvent {
private final String threadId;
public QueuedRemoveEvent(Object source, String threadId) {
super(source);
this.threadId = threadId;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class QueuedUpdateEvent extends ApplicationEvent {
private final Long id;
public QueuedUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
super(message);
}
public DownloadException(Exception e) {
public DownloadException(Throwable e) {
super(e);
}
}
@@ -1,7 +1,7 @@
package tn.mnlr.vripper.exception;
public class HostException extends Exception {
public HostException(Exception e) {
public HostException(Throwable e) {
super(e);
}
@@ -9,7 +9,7 @@ public class HostException extends Exception {
super(message);
}
public HostException(String message, Exception e) {
super(message,e);
public HostException(String message, Throwable e) {
super(message, e);
}
}
@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
public class HtmlProcessorException extends Exception {
public HtmlProcessorException(Exception e) {
public HtmlProcessorException(Throwable e) {
super(e);
}
}
@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
super(message);
}
public PostParseException(String message, Exception e) {
public PostParseException(String message, Throwable e) {
super(message, e);
}
public PostParseException(Exception e) {
public PostParseException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,16 @@
package tn.mnlr.vripper.exception;
public class QueueException extends Exception {
public QueueException(String message) {
super(message);
}
public QueueException(String message, Throwable e) {
super(message, e);
}
public QueueException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,10 @@
package tn.mnlr.vripper.exception;
import java.io.IOException;
public class RenameException extends Exception {
public RenameException(String message, IOException e) {
super(message, e);
}
}
@@ -5,7 +5,7 @@ public class VripperException extends Exception {
super(message);
}
public VripperException(Exception e) {
public VripperException(Throwable e) {
super(e);
}
}
@@ -1,7 +1,7 @@
package tn.mnlr.vripper.exception;
public class XpathException extends Exception {
public XpathException(Exception e) {
public XpathException(Throwable e) {
super(e);
}
}
@@ -1,37 +1,49 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class AcidimgHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(AcidimgHost.class);
private static final String host = "acidimg.cc";
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
public static final String IMG_XPATH = "//img[@class='centred']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final ConnectionService cm;
private final HostService hostService;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public AcidimgHost(ConnectionService cm, HostService hostService, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.hostService = hostService;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -39,22 +51,27 @@ public class AcidimgHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node contDiv;
try {
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.info(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
HttpPost httpPost = cm.buildHttpPost(url, context);
httpPost.addHeader("Referer", url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
@@ -64,9 +81,9 @@ public class AcidimgHost extends Host {
throw new HostException(e);
}
logger.info(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
logger.info(String.format("Cleaning response for %s", httpPost));
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
@@ -76,23 +93,22 @@ public class AcidimgHost extends Host {
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if(imgNode == null) {
if (imgNode == null) {
throw new HostException("Cannot find the image node");
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? getDefaultImageName(imgUrl) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -1,226 +1,38 @@
package tn.mnlr.vripper.host;
import lombok.Getter;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.HostService;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Iterator;
import java.util.Random;
import java.util.Objects;
@Service
@Slf4j
abstract public class Host {
private static final Logger logger = LoggerFactory.getLogger(Host.class);
private static final int READ_BUFFER_SIZE = 8192;
@Autowired
protected HtmlProcessorService htmlProcessorService;
@Autowired
protected XpathService xpathService;
@Autowired
private AppSettingsService appSettingsService;
@Autowired
private ConnectionManager cm;
@Autowired
private DownloadSpeedService downloadSpeedService;
@Autowired
private PathService pathService;
abstract public String getHost();
abstract public String getLookup();
public boolean isSupported(String url) {
return url.contains(getHost());
return url.contains(getLookup());
}
public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
public abstract HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException;
try {
imageFileData.setPageUrl(image.getUrl());
/**
* HOST SPECIFIC
*/
logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
setNameAndUrl(image.getUrl(), imageFileData);
logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
logger.info(String.format("Building image request for %s", image.getUrl()));
setImageRequest(imageFileData);
/**
* END HOST SPECIFIC
*/
String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
logger.info(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
imageFileData.setImageName(formatImageFileName);
File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
logger.info(String.format("Saving to %s", destinationFolder.getPath()));
if (!destinationFolder.exists()) {
logger.info(String.format("Creating %s", destinationFolder.getPath()));
destinationFolder.mkdirs();
}
HttpClient client = cm.getClient().build();
logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
if(response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
InputStream downloadStream = response.getEntity().getContent();
FileOutputStream fos = new FileOutputStream(outputFile);
try {
image.setTotal(response.getEntity().getContentLength());
logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
}
EntityUtils.consumeQuietly(response.getEntity());
} finally {
if (downloadStream != null) {
downloadStream.close();
}
if (fos != null) {
fos.close();
}
}
checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
}
} catch (Exception e) {
if(Thread.interrupted()) {
throw new InterruptedException("Download was interrupted");
}
throw new DownloadException(e);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Host host = (Host) o;
return Objects.equals(getHost(), host.getHost());
}
private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
String formatName;
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
formatName = reader.getFormatName();
if (formatName.toUpperCase().equals("JPEG")) {
formatName = "jpg";
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
if (outImage.exists()) {
outImage.delete();
}
Files.move(outputFile.toPath(), outImage.toPath());
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
}
}
/**
* Just for testing, you may ignore
* @throws Exception
*/
private void randomFail() throws Exception {
Random random = new Random();
int i = random.nextInt(100);
if (i < 50) {
throw new Exception("Purpose error");
}
}
protected final String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
protected final Response getResponse(final String url) throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
Header[] headers;
logger.info(String.format("Requesting %s", url));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
logger.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
protected void setImageRequest(final ImageFileData imageFileData) {
HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
}
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
@Getter
public static class Response {
protected Response(Document document, Header[] headers) {
this.document = document;
this.headers = headers;
}
private Document document;
private Header[] headers;
@Override
public int hashCode() {
return Objects.hash(getHost());
}
@Override
@@ -1,12 +1,11 @@
package tn.mnlr.vripper.host;
import org.apache.http.Header;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -14,24 +13,33 @@ import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.io.IOException;
import java.util.Arrays;
import java.util.stream.Collectors;
@Service
@Slf4j
public class ImageBamHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
private static final String host = "imagebam.com";
public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
public static final String IMG_XPATH = "//img[@class='image']";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
private static final String IMG_XPATH = "//img[@class='image']";
private final HostService hostService;
private final ConnectionService cm;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public ImageBamHost(HostService hostService, ConnectionService cm, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
this.hostService = hostService;
this.cm = cm;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -39,37 +47,34 @@ public class ImageBamHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Response response = getResponse(url);
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
HostService.Response response = hostService.getResponse(url, context);
Document doc = response.getDocument();
Header[] headers = response.getHeaders();
String cookies = Arrays.asList(headers)
.stream()
.filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
.map(e -> e.getValue().split(";")[0].trim())
.collect(Collectors.joining("; "));
Node contDiv;
try {
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.info(String.format("Getting cookies to use for %s", url));
logger.info(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
HttpGet httpGet = cm.buildHttpGet(url, context);
httpGet.addHeader("Referer", url);
httpGet.addHeader("Cookie", cookies);
logger.info(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
log.debug(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
String s = EntityUtils.toString(res.getEntity());
logger.debug(String.format("%s response is:%n%s", httpGet, s));
logger.debug(String.format("Cleaning response for %s", httpGet));
log.debug(String.format("%s response is:%n%s", httpGet, s));
log.debug(String.format("Cleaning response for %s", httpGet));
doc = htmlProcessorService.clean(s);
EntityUtils.consumeQuietly(res.getEntity());
} catch (IOException | HtmlProcessorException e) {
@@ -79,20 +84,19 @@ public class ImageBamHost extends Host {
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
} catch(Exception e) {
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
@@ -1,26 +1,33 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class ImageTwistHost extends Host {
public static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "imagetwist.com";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public ImageTwistHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,25 +35,29 @@ public class ImageTwistHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,82 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImageVenueHost extends Host {
private static final String host = "imagevenue.com";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageVenueHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
//Button detected. No need to actually click it, just make the call again.
resp = hostService.getResponse(url, context);
doc = resp.getDocument();
}
} catch (XpathException e) {
throw new HostException(e);
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,26 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImageZillaHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
private static final String host = "imagezilla.net";
public static final String IMG_XPATH = "//img[@id='photo']";
private static final String lookup = "imagezilla.net/show";
private static final String IMG_XPATH = "//img[@id='photo']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public ImageZillaHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,16 +34,21 @@ public class ImageZillaHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return lookup;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
String title;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
logger.info(String.format("Resolving name for %s", url));
if(titleNode != null) {
log.debug(String.format("Resolving name for %s", url));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
@@ -46,13 +57,12 @@ public class ImageZillaHost extends Host {
throw new HostException(e);
}
if(title == null || title.isEmpty()) {
title = getDefaultImageName(url);
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(url);
}
try {
imageFileData.setImageUrl(url.replace("show", "images"));
imageFileData.setImageName(title);
return new HostService.NameUrl(title, url.replace("show", "images"));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,65 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImgSpiceHost extends Host {
private static final String host = "imgspice.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,26 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImgboxHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
private static final String host = "imgbox.com";
public static final String IMG_XPATH = "//img[@id='img']";
private static final String IMG_XPATH = "//img[@id='img']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public ImgboxHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,25 +34,29 @@ public class ImgboxHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -1,14 +1,14 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -16,24 +16,35 @@ import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class ImxHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
private static final String host = "imx.to";
public static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
public static final String IMG_XPATH = "//img[@class='centred']";
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final HostService hostService;
private final XpathService xpathService;
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public ImxHost(HostService hostService, XpathService xpathService, ConnectionService cm, HtmlProcessorService htmlProcessorService) {
this.hostService = hostService;
this.xpathService = xpathService;
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -41,56 +52,67 @@ public class ImxHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node contDiv;
String value = null;
try {
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
Node node = contDiv.getAttributes().getNamedItem("value");
if (node != null) {
value = node.getTextContent();
}
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.info(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
httpPost.addHeader("Referer", url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to image ... "));
try {
httpPost.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
throw new HostException(e);
}
logger.info(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
logger.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
throw new HostException(e);
}
if (value == null) {
throw new HostException("Failed to obtain value attribute from continue input");
}
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url, context);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", value));
try {
httpPost.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
throw new HostException(e);
}
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
throw new HostException(e);
}
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
} catch(Exception e) {
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
@@ -0,0 +1,75 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PimpandhostHost extends Host {
private static final String host = "pimpandhost.com";
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PimpandhostHost(HostService hostService, XpathService xpathService) {
super();
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url;
try {
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
} catch (Exception e) {
throw new HostException(e);
}
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle,
imgUrl
);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,69 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PixRouteHost extends Host {
private static final String host = "pixroute.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixRouteHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
return new HostService.NameUrl(
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
imgNode.getAttributes().getNamedItem("src").getTextContent().trim()
);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,26 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PixhostHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
private static final String host = "pixhost.to";
public static final String IMG_XPATH = "//img[@id='image']";
private static final String lookup = "pixhost.to/show";
private static final String IMG_XPATH = "//img[@id='image']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public PixhostHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,25 +34,29 @@ public class PixhostHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return lookup;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1));
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PixxxelsHost extends Host {
private static final String host = "pixxxels.cc";
private static final String IMG_XPATH = "//*[@id='download']";
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixxxelsHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = titleNode.getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,69 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class PostImgHost extends Host {
private static final String host = "postimg.cc";
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
private static final String IMG_XPATH = "//a[@id='download']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PostImgHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node urlNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> hostService.getDefaultImageName(url));
return new HostService.NameUrl(imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,27 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class TurboImageHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
private static final String host = "turboimagehost.com";
public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
public static final String IMG_XPATH = "//img[@id='uImage']";
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
private static final String IMG_XPATH = "//img[@id='imageid']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public TurboImageHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -29,16 +34,21 @@ public class TurboImageHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
String title;
try {
logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
logger.info(String.format("Resolving name for %s", url));
if(titleNode != null) {
log.debug(String.format("Resolving name for %s", url));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
@@ -47,14 +57,13 @@ public class TurboImageHost extends Host {
throw new HostException(e);
}
if(title == null || title.isEmpty()) {
title = getDefaultImageName(url);
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(url);
}
try {
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
imageFileData.setImageName(title);
return new HostService.NameUrl(title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,65 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class ViprImHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "vipr.im";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ViprImHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.jpa;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.services.ThreadPoolService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
@EnableScheduling
public class Management {
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Pattern BACKUP_FILE_PATTERN = Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
private final JdbcTemplate jdbcTemplate;
private final String backupFolder;
private final ThreadPoolService threadPoolService;
private final DownloadService downloadService;
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName, ThreadPoolService threadPoolService, DownloadService downloadService) {
this.jdbcTemplate = jdbcTemplate;
this.threadPoolService = threadPoolService;
this.downloadService = downloadService;
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
}
@PreDestroy
public void destroy() {
try {
threadPoolService.destroy();
downloadService.destroy();
this.jdbcTemplate.execute("SHUTDOWN");
} catch (Exception ignored) {
Thread.currentThread().interrupt();
}
}
@PostConstruct
@Scheduled(cron = "0 0 0 ? * *")
private void backup() {
for (File file : Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[]{})) {
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
if (matcher.find()) {
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
if (localDate.isEqual(LocalDate.now())) {
return;
}
}
}
// create a backup file
File backupFile = new File(backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
}
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Objects;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Image {
@JsonIgnore
protected Long id;
@JsonIgnore
private Host host;
private String url;
private int index;
private long current = 0;
private long total = 0;
private Status status;
private String postId;
@JsonIgnore
private Long postIdRef;
public Image(String postId, String url, Host host, int index) {
this.postId = postId;
this.url = url;
this.host = host;
this.index = index;
status = Status.STOPPED;
}
public void increase(int read) {
current += read;
}
public void init() {
current = 0;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return Objects.equals(url, image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -0,0 +1,24 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.Collections;
import java.util.List;
@Getter
@Setter
@NoArgsConstructor
public class Metadata {
@JsonIgnore
private Long id;
private List<String> resolvedNames = Collections.emptyList();
private String postedBy;
private Long postIdRef;
}
@@ -0,0 +1,78 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Post {
@JsonIgnore
private Long id;
private Status status;
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private int done;
private int total;
private Set<String> hosts;
private String forum;
@JsonIgnore
private String securityToken;
@JsonIgnore
private String downloadDirectory;
private boolean thanked;
private Set<String> previews = Collections.emptySet();
private Metadata metadata;
public Post(String title, String url, String postId, String threadId, String threadTitle, String forum, String securityToken) {
this.title = title;
this.url = url;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return Objects.equals(postId, post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -0,0 +1,57 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.util.Objects;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class Queued {
@JsonIgnore
private Long id;
private String link;
private String threadId;
private String postId;
private int total = 0;
private boolean loading = true;
public Queued(String link, String threadId, String postId) {
this();
this.link = link;
this.threadId = threadId;
this.postId = postId;
}
public void done() {
this.loading = false;
}
public void increment() {
this.total++;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Queued that = (Queued) o;
return Objects.equals(threadId, that.threadId);
}
@Override
public int hashCode() {
return Objects.hash(threadId);
}
}
@@ -0,0 +1,5 @@
package tn.mnlr.vripper.jpa.domain.enums;
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED, PARTIAL
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.List;
import java.util.Optional;
public interface IImageRepository extends IRepository {
Image save(Image image);
void deleteAllByPostId(String postId);
List<Image> findByPostId(String postId);
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
int stopByPostIdAndIsNotCompleted(String postId);
List<Image> findByPostIdAndIsError(String postId);
Optional<Image> findById(Long id);
int updateStatus(Status status, Long id);
int updateCurrent(long current, Long id);
int updateTotal(long total, Long id);
}
@@ -0,0 +1,16 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Metadata;
import java.util.Optional;
public interface IMetadataRepository extends IRepository {
Metadata save(Metadata metadata);
Optional<Metadata> findById(Long id);
Optional<Metadata> findByPostId(String postId);
int deleteByPostId(String postId);
}
@@ -0,0 +1,36 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.List;
import java.util.Optional;
public interface IPostRepository extends IRepository {
Post save(Post post);
Optional<Post> findByPostId(String postId);
Optional<Post> findById(Long id);
List<String> findCompleted();
List<Post> findAll();
boolean existByPostId(String postId);
int setDownloadingToStopped();
int deleteByPostId(String postId);
int updateStatus(Status status, Long id);
int updateDone(int done, Long id);
int updateFolderName(String postFolderName, Long id);
int updateTitle(String title, Long id);
int updateThanked(boolean thanked, Long id);
}
@@ -0,0 +1,18 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.util.List;
import java.util.Optional;
public interface IQueuedRepository extends IRepository {
Queued save(Queued queued);
Optional<Queued> findByThreadId(String threadId);
List<Queued> findAll();
Optional<Queued> findById(Long id);
int deleteByThreadId(String threadId);
}
@@ -0,0 +1,6 @@
package tn.mnlr.vripper.jpa.repositories;
public interface IRepository {
void init();
}
@@ -0,0 +1,153 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.ImageUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public ImageRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM IMAGE",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Image save(Image image) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
id,
image.getCurrent(),
image.getHost().getHost(),
image.getIndex(),
image.getPostId(),
image.getStatus().name(),
image.getTotal(),
image.getUrl(),
image.getPostIdRef()
);
image.setId(id);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return image;
}
@Override
public void deleteAllByPostId(String postId) {
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
}
@Override
public List<Image> findByPostId(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE WHERE POST_ID = ?",
new ImageRowMapper(),
postId);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'",
Integer.class
);
}
@Override
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
new ImageRowMapper(),
postId);
}
@Override
public int stopByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
postId
);
}
@Override
public List<Image> findByPostIdAndIsError(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
new ImageRowMapper(),
postId);
}
@Override
public Optional<Image> findById(Long id) {
List<Image> images = jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.ID = ?",
new ImageRowMapper(),
id);
if (images.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(images.get(0));
}
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?",
status.name(), id
);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return mutationCount;
}
@Override
public int updateCurrent(long current, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?",
current, id
);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return mutationCount;
}
@Override
public int updateTotal(long total, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?",
total, id
);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return mutationCount;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,29 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.sql.ResultSet;
import java.sql.SQLException;
public class ImageRowMapper implements RowMapper<Image> {
@Override
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
Image image = new Image();
image.setId(rs.getLong("ID"));
String host = rs.getString("HOST");
image.setHost(SpringContext.getBeansOfType(Host.class).values().stream().filter(e -> e.getHost().equals(host)).findAny().orElse(null));
image.setUrl(rs.getString("URL"));
image.setIndex(rs.getInt("INDEX"));
image.setCurrent(rs.getLong("CURRENT"));
image.setTotal(rs.getLong("TOTAL"));
image.setStatus(Status.valueOf(rs.getString("STATUS")));
image.setPostId(rs.getString("POST_ID"));
image.setPostIdRef(rs.getLong("POST_ID_REF"));
return image;
}
}
@@ -0,0 +1,95 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.MetadataUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public MetadataRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM IMAGE",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Metadata save(Metadata metadata) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO METADATA (ID, POSTED_BY, RESOLVED_NAMES, POST_ID_REF) VALUES (?,?,?,?)",
id,
metadata.getPostedBy(),
String.join("%sep%", metadata.getResolvedNames()),
metadata.getPostIdRef()
);
metadata.setId(id);
applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, id, metadata.getPostIdRef()));
return metadata;
}
@Override
public Optional<Metadata> findById(Long id) {
List<Metadata> metadata = jdbcTemplate.query(
"SELECT * FROM METADATA AS metadata WHERE metadata.ID = ?",
new Object[]{id},
new MetadataRowMapper()
);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public Optional<Metadata> findByPostId(String postId) {
List<Metadata> metadata = jdbcTemplate.query(
"SELECT metadata.* FROM METADATA AS metadata INNER JOIN POST post ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?",
new Object[]{postId},
new MetadataRowMapper()
);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM METADATA AS metadata WHERE metadata.ID = (SELECT inner_metadata.ID FROM POST AS post INNER JOIN METADATA inner_metadata ON post.ID = inner_metadata.POST_ID_REF WHERE post.POST_ID = ?)",
postId
);
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,24 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Metadata;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
public class MetadataRowMapper implements RowMapper<Metadata> {
@Override
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
Metadata metadata = new Metadata();
metadata.setId(rs.getLong("ID"));
metadata.setPostedBy(rs.getString("POSTED_BY"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
return metadata;
}
}
@@ -0,0 +1,195 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.PostRemoveEvent;
import tn.mnlr.vripper.event.PostUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class PostRepository implements IPostRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public PostRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM POST",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Post save(Post post) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
String.join(";", post.getHosts()),
post.getDownloadDirectory(),
post.getPostId(),
String.join(";", post.getPreviews()),
post.getSecurityToken(),
post.getStatus().name(),
post.isThanked(),
post.getThreadId(),
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
post.getUrl()
);
post.setId(id);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return post;
}
@Override
public Optional<Post> findByPostId(String postId) {
List<Post> posts = jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE POST_ID = ?",
new PostRowMapper(),
postId);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<String> findCompleted() {
return jdbcTemplate.query(
"SELECT POST_ID FROM POST WHERE status = 'COMPLETE' AND done >= total",
((rs, rowNum) -> rs.getString("POST_ID"))
);
}
@Override
public Optional<Post> findById(Long id) {
List<Post> posts = jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
new PostRowMapper(),
id);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<Post> findAll() {
return jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
new PostRowMapper()
);
}
@Override
public boolean existByPostId(String postId) {
Integer count = jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?",
Integer.class,
postId);
if (count == null) {
return false;
} else {
return count > 0;
}
}
@Override
public int setDownloadingToStopped() {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'"
);
}
@Override
public int deleteByPostId(String postId) {
int mutationCount = jdbcTemplate.update(
"DELETE FROM POST AS post WHERE post.POST_ID = ?",
postId
);
applicationEventPublisher.publishEvent(new PostRemoveEvent(PostRepository.class, postId));
return mutationCount;
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?",
status.name(), id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateDone(int done, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?",
done, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateFolderName(String postFolderName, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
postFolderName, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateTitle(String title, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?",
title, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateThanked(boolean thanked, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?",
thanked, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,55 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Set;
public class PostRowMapper implements RowMapper<Post> {
private static final String DELIMITER = ";";
@Override
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
Post post = new Post();
post.setId(rs.getLong("post.ID"));
post.setStatus(Status.valueOf(rs.getString("STATUS")));
post.setPostId(rs.getString("POST_ID"));
post.setThreadTitle(rs.getString("THREAD_TITLE"));
post.setThreadId(rs.getString("THREAD_ID"));
post.setTitle(rs.getString("TITLE"));
post.setUrl(rs.getString("URL"));
post.setDone(rs.getInt("DONE"));
post.setTotal(rs.getInt("TOTAL"));
post.setHosts(Set.of(rs.getString("HOSTS").split(DELIMITER)));
post.setForum(rs.getString("FORUM"));
post.setSecurityToken(rs.getString("SECURITY_TOKEN"));
post.setDownloadDirectory(rs.getString("POST_FOLDER_NAME"));
post.setThanked(rs.getBoolean("THANKED"));
String previews;
if ((previews = rs.getString("PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
Long metadataId = rs.getLong("metadata.ID");
if (!rs.wasNull()) {
Metadata metadata = new Metadata();
metadata.setId(metadataId);
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostedBy(rs.getString("POSTED_BY"));
post.setMetadata(metadata);
}
return post;
}
}
@@ -0,0 +1,108 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.QueuedRemoveEvent;
import tn.mnlr.vripper.event.QueuedUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
@Service
public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private final AtomicLong counter = new AtomicLong(0);
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public QueuedRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void init() {
Long maxId = jdbcTemplate.queryForObject(
"SELECT MAX(ID) FROM QUEUED",
Long.class
);
if (maxId == null) {
maxId = 0L;
}
counter.set(maxId);
}
@Override
public Queued save(Queued queued) {
long id = counter.incrementAndGet();
jdbcTemplate.update(
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
id,
queued.getTotal(),
queued.getLink(),
queued.isLoading(),
queued.getPostId(),
queued.getThreadId()
);
queued.setId(id);
applicationEventPublisher.publishEvent(new QueuedUpdateEvent(QueuedRepository.class, id));
return queued;
}
@Override
public Optional<Queued> findByThreadId(String threadId) {
List<Queued> queuedList = jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
new Object[]{threadId},
new QueuedRowMapper()
);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public List<Queued> findAll() {
return jdbcTemplate.query(
"SELECT * FROM QUEUED",
new QueuedRowMapper()
);
}
@Override
public Optional<Queued> findById(Long id) {
List<Queued> queuedList = jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?",
new Object[]{id},
new QueuedRowMapper()
);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public int deleteByThreadId(String threadId) {
int mutationCount = jdbcTemplate.update(
"DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?",
threadId
);
applicationEventPublisher.publishEvent(new QueuedRemoveEvent(QueuedRepository.class, threadId));
return mutationCount;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,22 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.sql.ResultSet;
import java.sql.SQLException;
public class QueuedRowMapper implements RowMapper<Queued> {
@Override
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
Queued queued = new Queued();
queued.setId(rs.getLong("ID"));
queued.setLink(rs.getString("LINK"));
queued.setThreadId(rs.getString("THREAD_ID"));
queued.setPostId(rs.getString("POST_ID"));
queued.setTotal(rs.getInt("TOTAL"));
queued.setLoading(rs.getBoolean("LOADING"));
return queued;
}
}
@@ -0,0 +1,7 @@
package tn.mnlr.vripper.listener;
import reactor.core.publisher.Flux;
public interface DataEventListener<T> {
Flux<T> getDataFlux();
}
@@ -0,0 +1,7 @@
package tn.mnlr.vripper.listener;
import reactor.core.publisher.Sinks.EmitFailureHandler;
public class EmitHandler {
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.ImageUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class ImageUpdateEventListener implements ApplicationListener<ImageUpdateEvent>, DataEventListener<ImageUpdateEvent> {
private final Sinks.Many<ImageUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(ImageUpdateEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<ImageUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.MetadataUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class MetadataUpdateEventListener implements ApplicationListener<MetadataUpdateEvent>, DataEventListener<MetadataUpdateEvent> {
private final Sinks.Many<MetadataUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(MetadataUpdateEvent event) {
sink.emitNext(event, (signalType, emitResult) -> true);
}
@Override
public Flux<MetadataUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.PostRemoveEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class PostRemoveEventListener implements ApplicationListener<PostRemoveEvent>, DataEventListener<PostRemoveEvent> {
private final Sinks.Many<PostRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(PostRemoveEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<PostRemoveEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.PostUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class PostUpdateEventListener implements ApplicationListener<PostUpdateEvent>, DataEventListener<PostUpdateEvent> {
private final Sinks.Many<PostUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(PostUpdateEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<PostUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.QueuedRemoveEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class QueuedRemoveEventListener implements ApplicationListener<QueuedRemoveEvent>, DataEventListener<QueuedRemoveEvent> {
private final Sinks.Many<QueuedRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(QueuedRemoveEvent event) {
sink.emitNext(event, (signalType, emitResult) -> true);
}
@Override
public Flux<QueuedRemoveEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.QueuedUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class QueuedUpdateEventListener implements ApplicationListener<QueuedUpdateEvent>, DataEventListener<QueuedUpdateEvent> {
private final Sinks.Many<QueuedUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(QueuedUpdateEvent event) {
sink.emitNext(event, (signalType, emitResult) -> true);
}
@Override
public Flux<QueuedUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -1,48 +0,0 @@
package tn.mnlr.vripper.q;
import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tn.mnlr.vripper.entities.Image;
import java.util.Objects;
import java.util.concurrent.Callable;
public class DownloadJob implements Callable<Image> {
private static final Logger logger = LoggerFactory.getLogger(DownloadJob.class);
@Getter
private final Image image;
@Getter
private final ImageFileData imageFileData = new ImageFileData();
public DownloadJob(Image image) {
this.image = image;
}
@Override
public Image call() throws Exception {
logger.info(String.format("Starting downloading %s", image.getUrl()));
image.setStatus(Image.Status.DOWNLOADING);
image.setCurrent(0);
image.getHost().download(image, imageFileData);
image.setStatus(Image.Status.COMPLETE);
return image;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return image.equals(that.image);
}
@Override
public int hashCode() {
return Objects.hash(image);
}
}
@@ -1,156 +0,0 @@
package tn.mnlr.vripper.q;
import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.entities.Post;
import tn.mnlr.vripper.services.AppStateService;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.stream.Collectors;
@Service
public class DownloadQ {
private static final List<Post.Status> FINISHED = Arrays.asList(Post.Status.ERROR, Post.Status.COMPLETE, Post.Status.STOPPED);
private static final Logger logger = LoggerFactory.getLogger(DownloadQ.class);
@Autowired
private AppStateService appStateService;
@Autowired
private ExecutionService executionService;
private BlockingQueue<DownloadJob> downloadQ = new LinkedBlockingQueue<>();
@Getter
private boolean notPauseQ = true;
public void put(Image image) throws InterruptedException {
synchronized (appStateService) {
logger.info(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
DownloadJob downloadJob = new DownloadJob(image);
downloadQ.put(downloadJob);
appStateService.newDownloadJob(downloadJob);
}
}
public DownloadJob take() throws InterruptedException {
DownloadJob downloadJob = downloadQ.take();
logger.info(String.format("Retrieving a job for %s", downloadJob.getImage().getUrl()));
return downloadJob;
}
public synchronized void enqueue(Post post) throws InterruptedException {
for (Image image : post.getImages()) {
put(image);
}
}
public void restart(String postId) throws InterruptedException {
synchronized (appStateService) {
if (appStateService.getRunningPosts().get(postId) != null && appStateService.getRunningPosts().get(postId).get() > 0) {
logger.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
List<Image> images = appStateService.getPost(postId)
.getImages()
.stream()
.filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
.collect(Collectors.toList());
if (images.isEmpty()) {
return;
}
appStateService.getPost(postId).setStatus(Post.Status.PENDING);
logger.info(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
put(image);
}
}
}
public void removeScheduled(Image image) {
synchronized (appStateService) {
image.setStatus(Image.Status.STOPPED);
logger.info(String.format("Removing scheduled job for %s", image.getUrl()));
Iterator<DownloadJob> iterator = downloadQ.iterator();
boolean removed = false;
while (iterator.hasNext()) {
DownloadJob next = iterator.next();
if (next.getImage().getPostId().equals(image.getPostId())) {
iterator.remove();
appStateService.doneDownloadJob(image);
logger.info(String.format("Scheduled job for %s is removed", image.getUrl()));
removed = true;
break;
}
}
if (!removed) {
logger.warn(String.format("Job for %s does not exist", image.getUrl()));
}
image.cleanup();
}
}
public void removeRunning(String postId) {
logger.info(String.format("Interrupting running jobs for post id %s", postId));
executionService.stop(postId);
}
public void stop(String postId) {
try {
synchronized (appStateService) {
if (FINISHED.contains(appStateService.getPost(postId).getStatus())) {
return;
}
notPauseQ = false;
appStateService.getPost(postId).setStatus(Post.Status.STOPPED);
List<Image> images = appStateService.getPost(postId)
.getImages()
.stream()
.filter(e -> !e.getStatus().equals(Image.Status.COMPLETE))
.collect(Collectors.toList());
if (images.isEmpty()) {
return;
}
logger.info(String.format("Stopping %d jobs for post id %s", images.size(), postId));
images.forEach(image -> removeScheduled(image));
removeRunning(postId);
}
} finally {
notPauseQ = true;
}
}
public int size() {
return downloadQ.size();
}
public void stopAll() {
synchronized (appStateService) {
appStateService.getCurrentPosts().values().stream().map(Post::getPostId).forEach(this::stop);
}
}
public void restartAll() throws InterruptedException {
synchronized (appStateService) {
for (Post post : appStateService.getCurrentPosts().values()) {
String postId = post.getPostId();
restart(postId);
}
}
}
}
@@ -1,160 +0,0 @@
package tn.mnlr.vripper.q;
import net.jodah.failsafe.Failsafe;
import net.jodah.failsafe.FailsafeException;
import net.jodah.failsafe.RetryPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.AppStateService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Service
public class ExecutionService {
private static final Logger logger = LoggerFactory.getLogger(ExecutionService.class);
@Autowired
private DownloadQ downloadQ;
@Autowired
private AppSettingsService settings;
@Autowired
private AppStateService appStateService;
private AtomicInteger threadCount = new AtomicInteger();
private ExecutorService executor = Executors.newFixedThreadPool(10);
private Thread executionThread;
private RetryPolicy<Object> retryPolicy;
private List<DownloadJob> running = Collections.synchronizedList(new ArrayList<>());
private Map<String, Future<?>> futures = new ConcurrentHashMap<>();
@PostConstruct
private void init() {
retryPolicy = new RetryPolicy<>()
.handleIf(e -> !(e instanceof InterruptedException))
.withDelay(Duration.ofSeconds(5))
.withMaxRetries(2)
.abortOn(InterruptedException.class)
.onFailedAttempt(e -> logger.warn(String.format("#%d tries failed", e.getAttemptCount()), e.getLastFailure()));
executionThread = new Thread(this::start, "Executor thread");
executionThread.start();
}
@PreDestroy
private void destroy() throws InterruptedException {
logger.info("Shutting down ExecutionService");
executionThread.interrupt();
executor.shutdown();
appStateService.getCurrentPosts().keySet().forEach(p -> {
logger.info(String.format("Stopping download jobs for %s", p));
this.stop(p);
});
executor.awaitTermination(10, TimeUnit.SECONDS);
}
public void stop(String postId) {
List<DownloadJob> data = running
.stream()
.filter(e -> e.getImage().getPostId().equals(postId))
.peek(e -> e.getImage().setStatus(Image.Status.STOPPED))
.collect(Collectors.toList());
logger.warn(String.format("Interrupting %d jobs for post id %s", data.size(), postId));
data.forEach(e -> {
futures.get(e.getImage().getUrl()).cancel(true);
if(e.getImageFileData().getImageRequest() != null) {
e.getImageFileData().getImageRequest().abort();
}
e.getImage().cleanup();
});
}
boolean canRun() {
boolean canRun = threadCount.get() < settings.getMaxThreads();
if (canRun && downloadQ.isNotPauseQ()) {
threadCount.incrementAndGet();
return true;
}
return false;
}
public void start() {
while (!Thread.interrupted()) {
if (canRun()) {
DownloadJob take = null;
try {
take = downloadQ.take();
if (take == null) {
continue;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
DownloadJob finalTake = take;
Runnable task = () -> {
running.add(finalTake);
Failsafe.with(retryPolicy)
.onFailure(e -> {
if (e.getFailure() instanceof InterruptedException || (e.getFailure() instanceof FailsafeException && e.getFailure().getCause() instanceof InterruptedException)) {
logger.info("Job successfully interrupted");
return;
}
logger.error(String.format("Failed to download %s after %d tries", finalTake.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
finalTake.getImage().setStatus(Image.Status.ERROR);
})
.onComplete(e -> {
appStateService.doneDownloadJob(finalTake.getImage());
logger.info(String.format("Finished downloading %s", finalTake.getImage().getUrl()));
synchronized (threadCount) {
threadCount.decrementAndGet();
running.remove(finalTake);
futures.remove(finalTake.getImage().getUrl());
threadCount.notify();
}
})
.get(finalTake::call);
};
logger.info(String.format("Scheduling a job for %s", finalTake.getImage().getUrl()));
futures.put(finalTake.getImage().getUrl(), executor.submit(task));
} else {
synchronized (threadCount) {
try {
threadCount.wait(2_000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
}
}
public int runningCount() {
return running.size();
}
}
@@ -1,15 +0,0 @@
package tn.mnlr.vripper.q;
import lombok.Getter;
import lombok.Setter;
import org.apache.http.client.methods.HttpUriRequest;
@Getter
@Setter
public class ImageFileData {
private String pageUrl;
private String imageName;
private String imageUrl;
private HttpUriRequest imageRequest;
}
@@ -1,169 +0,0 @@
package tn.mnlr.vripper.services;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.exception.ValidationException;
import javax.annotation.PreDestroy;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.prefs.BackingStoreException;
import java.util.prefs.Preferences;
@Service
@Getter
@Setter
public class AppSettingsService {
private static final Logger logger = LoggerFactory.getLogger(AppSettingsService.class);
private Preferences prefs = Preferences.userNodeForPackage(AppSettingsService.class);
private final String DOWNLOAD_PATH = "DOWNLOAD_PATH";
private final String MAX_THREADS = "MAX_THREADS";
private final String AUTO_START = "AUTO_START";
private final String V_LOGIN = "VLOGIN";
private final String V_USERNAME = "VUSERNAME";
private final String V_PASSWORD = "VPASSWORD";
private final String V_THANKS = "VTHANKS";
private final String DESKTOP_CLIPBOARD = "DESKTOP_CLIPBOARD";
private final String FORCE_ORDER = "FORCE_ORDER";
private final String DARK_THEME = "DARK_THEME";
private String downloadPath;
private int maxThreads;
private boolean autoStart;
private boolean vLogin;
private String vUsername;
private String vPassword;
private boolean vThanks;
private boolean desktopClipboard;
private boolean forceOrder;
private boolean darkTheme;
public void setVPassword(String vPassword) {
if(vPassword.isEmpty()) {
this.vPassword = "";
} else {
this.vPassword = DigestUtils.md5Hex(vPassword);
}
}
public void restore() {
downloadPath = prefs.get(DOWNLOAD_PATH, System.getProperty("user.dir"));
maxThreads = prefs.getInt(MAX_THREADS, 4);
autoStart = prefs.getBoolean(AUTO_START, true);
vLogin = prefs.getBoolean(V_LOGIN, false);
vUsername = prefs.get(V_USERNAME, "");
vPassword = prefs.get(V_PASSWORD, "");
vThanks = prefs.getBoolean(V_THANKS, false);
desktopClipboard = prefs.getBoolean(DESKTOP_CLIPBOARD, false);
forceOrder = prefs.getBoolean(FORCE_ORDER, false);
darkTheme = prefs.getBoolean(DARK_THEME, false);
}
@PreDestroy
public void save() {
prefs.put(DOWNLOAD_PATH, downloadPath);
prefs.putInt(MAX_THREADS, maxThreads);
prefs.putBoolean(AUTO_START, autoStart);
prefs.putBoolean(V_LOGIN, vLogin);
prefs.put(V_USERNAME, vUsername);
prefs.put(V_PASSWORD, vPassword);
prefs.putBoolean(V_THANKS, vThanks);
prefs.putBoolean(DESKTOP_CLIPBOARD, desktopClipboard);
prefs.putBoolean(FORCE_ORDER, forceOrder);
prefs.putBoolean(DARK_THEME, darkTheme);
try {
prefs.sync();
} catch (BackingStoreException e) {
logger.error("Failed to store user settings", e);
}
}
public void check(Settings settings) throws ValidationException {
Path path;
try {
path = Paths.get(settings.getDownloadPath());
} catch (InvalidPathException e) {
throw new ValidationException(String.format("%s is invalid", settings.getDownloadPath()));
}
if (!Files.exists(path)) {
throw new ValidationException(String.format("%s does not exist", settings.getDownloadPath()));
} else if (!Files.isDirectory(path)) {
throw new ValidationException(String.format("%s is not a directory", settings.getDownloadPath()));
}
if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 8) {
throw new ValidationException(String.format("Invalid max concurrent download settings, values must be in [%d,%d]", 1, 8));
}
}
public Theme getTheme() {
return new Theme(this.darkTheme);
}
public void setTheme(Theme theme) {
this.darkTheme = theme.darkTheme;
save();
}
@Getter
@NoArgsConstructor
public static class Theme {
@JsonProperty("darkTheme")
private boolean darkTheme;
public Theme(boolean darkTheme) {
this.darkTheme = darkTheme;
}
}
@Getter
public static class Settings {
@JsonProperty("downloadPath")
private String downloadPath;
@JsonProperty("maxThreads")
private int maxThreads;
@JsonProperty("autoStart")
private boolean autoStart;
@JsonProperty("vLogin")
private boolean vLogin;
@JsonProperty("vUsername")
private String vUsername;
@JsonProperty("vPassword")
private String vPassword;
@JsonProperty("vThanks")
private boolean vThanks;
@JsonProperty("desktopClipboard")
private boolean desktopClipboard;
@JsonProperty("forceOrder")
private boolean forceOrder;
public Settings(String downloadPath, int maxThreads, boolean autoStart, boolean vLogin, String vUsername, String vPassword, boolean vThanks, boolean desktopClipboard, boolean forceOrder) {
this.downloadPath = downloadPath;
this.maxThreads = maxThreads;
this.autoStart = autoStart;
this.vLogin = vLogin;
this.vUsername = vUsername;
this.vPassword = vPassword;
this.vThanks = vThanks;
this.desktopClipboard = desktopClipboard;
this.forceOrder = forceOrder;
}
}
}

Some files were not shown because too many files have changed in this diff Show More