Compare commits

...
161 Commits
Author SHA1 Message Date
death-claw 1c5b8a0397 v3.3.2 2021-03-11 19:05:09 +01:00
death-claw cc2ce9482d Fix a bug with event logs
Change event names and add some icons
2021-03-11 19:02:01 +01:00
death-claw 41f1775360 v3.3.1 2021-02-18 16:04:34 +01:00
death-claw 733f1a08ac Add about menu
Fix README.md
2021-02-18 16:01:06 +01:00
death-claw 0cb0ee0100 v3.3.0 2021-02-16 19:15:58 +01:00
death-claw cb88e39c83 Add support for proxies
Add a Event Logs
Fix bugs with the database
2021-02-16 19:13:02 +01:00
death-claw 6375c0aed6 v3.2.3 2021-01-13 20:15:13 +01:00
death-claw 40ca4cf86b Finished posts are not cleared from the UI 2021-01-13 20:11:47 +01:00
death-clawandGitHub b3a798388b Update README.md 2021-01-13 12:54:51 +01:00
death-claw 6e69b2bce8 v3.2.2 2021-01-13 09:51:56 +01:00
death-claw bb6dcd77b9 Set total download to max value 2021-01-13 09:49:38 +01:00
death-claw 42e1aa73e2 v3.2.1 2021-01-12 20:02:17 +01:00
death-claw 26fde22af2 Code reformat 2021-01-12 19:37:12 +01:00
death-claw 798f39d54f Add settings for connection timeout
Add settings for maximum attempts
Enhance posts rename logic
Dependencies update
Switch reactive from rx-java to reactor
2021-01-12 19:32:43 +01:00
death-claw 3a21dfac38 Remove unused font 2021-01-06 14:55:49 +01:00
death-claw e8355b88b9 v3.2 2020-11-16 19:40:36 +01:00
death-claw 379660236d Commit missing files 2020-11-16 19:31:59 +01:00
death-claw c86ee91c27 v3.1.0 2020-11-16 16:44:26 +01:00
death-claw ff9d253242 UI revamp
Fix bugs
2020-11-16 16:42:30 +01:00
death-claw 53b45e00ba v3.0.11 2020-10-31 15:09:19 +01:00
death-claw 247b12a525 Add support for vipr.im 2020-10-31 15:06:26 +01:00
death-claw 7a8d332e6e v3.0.10 2020-10-04 16:31:57 +01:00
death-claw 749a6f18d2 Fix bug on settings update 2020-10-04 16:27:26 +01:00
death-claw 4107048f12 v3.0.9 2020-09-27 10:14:08 +01:00
death-claw f5d9ab85a2 Revert to appending post id to download folders
Fix a bug that lead to adding duplicate posts
Remove old unused settings
2020-09-27 10:12:29 +01:00
death-claw b82ee7b494 v3.0.8 2020-08-22 19:01:08 +01:00
death-claw 8a31dfc012 Bug fix which breaks concurrent downloads 2020-08-22 18:59:44 +01:00
death-claw 734c1b72c7 v3.0.7 2020-08-22 16:45:43 +01:00
death-claw 7abb7df1f6 Fix MacOs build 2020-08-22 16:44:24 +01:00
death-claw 0a7fa99a64 v3.0.6 2020-08-18 08:33:57 +01:00
death-claw 691026c8a9 Fix TurboImageHost 2020-08-18 08:32:33 +01:00
death-claw 3ef004b947 v3.0.5 2020-08-17 20:06:12 +01:00
death-claw 1b22a93e28 Update packages
Host class refactoring
2020-08-17 20:04:39 +01:00
death-claw 2639df0aaa Update packages
Host class refactoring
2020-08-17 20:04:23 +01:00
death-claw d5ae5d57d1 v3.0.4 2020-08-16 16:11:11 +01:00
death-claw 537d76d7be Improve rename UI 2020-08-16 16:06:17 +01:00
death-claw c1247faacc v3.0.3 2020-08-16 11:29:01 +01:00
death-claw efcc685f30 Remove notification feature 2020-08-16 11:26:57 +01:00
death-claw c825b22ce4 Fix some bugs with MacOS 2020-08-16 10:58:19 +01:00
death-claw 6ef5d5474f Fix some bugs with MacOS 2020-08-16 10:20:56 +01:00
death-claw bd25801c6c Fix base dir configuration for MacOS 2020-08-16 00:18:18 +01:00
death-claw 377b93c1ac Fix base dir configuration for AppImage 2020-08-16 00:02:18 +01:00
death-claw d498d20f6b Set default download path to user's home 2020-08-15 23:23:43 +01:00
death-claw d18544d414 Fix bin location for mac 2020-08-15 23:10:58 +01:00
death-claw 3c872b79bd Update app data configuration 2020-08-15 23:00:48 +01:00
death-claw 37900cac49 v3.0.2 2020-08-15 15:38:53 +01:00
death-claw be57c76d93 Fix bugs with download queue 2020-08-15 15:36:47 +01:00
death-claw 8b52ad7858 v3.0.1 2020-08-03 18:48:45 +01:00
death-claw a0976bde02 Fix bugs with download queue 2020-08-03 18:43:42 +01:00
death-claw 15c97d5cfc v3.0.0 2020-08-03 15:06:36 +01:00
death-claw 905cbd5a35 Code refactoring
Fix bugs with rename gallery
2020-08-03 15:03:42 +01:00
death-clawandGitHub 5c0e2e6185 Merge pull request #12 from life-claw/imgVenueFix
Fixed ImageVenue
2020-07-30 20:56:41 +01:00
death-clawandGitHub 4836d2dacb Merge branch 'master' into imgVenueFix 2020-07-30 20:56:30 +01:00
death-claw bc81ec1ab5 Bug fixes
Code enhancements
2020-07-30 14:59:58 +01:00
death-claw 8a4d039a37 Bug fixes
Code enhancements
2020-07-30 14:00:56 +01:00
death-claw 468639264a Major rewrites of data persistence, websocket, and user interface 2020-07-26 15:10:09 +01:00
death-claw 9b649b470f Major rewrites of data persistence, websocket, and user interface 2020-07-26 14:45:04 +01:00
life-claw edd5cf2bf7 Fixed ImageVenue
Updated XPaths to match their current markup
And added HTTPs redirect, since they support that now
2020-07-17 16:57:51 -05:00
death-claw 67c605e640 v2.12.1 2020-05-22 15:55:05 +01:00
death-claw 8d4b878656 Fix alternative name bug 2020-05-22 15:54:11 +01:00
death-claw bf532dd1d0 v2.12.0 2020-05-21 16:43:03 +01:00
death-claw eb1a125e59 Use new user hash from the api 2020-05-21 16:41:16 +01:00
death-claw 256659a72d v2.11.7 2020-05-08 12:57:41 +01:00
death-claw dbf3aa7e7d Fix bug with paths creation
Move Open Link button
Rename rename all title
2020-05-08 12:56:11 +01:00
death-claw a75ce181ff v2.11.6 2020-05-04 21:07:16 +01:00
death-claw 37ae6b4ebf Merge branch 'master' of https://github.com/death-claw/vripper-project 2020-05-04 21:06:48 +01:00
death-claw 6631a6a95b PNG filenames are now correctly named
Fix issue with special characters in post titles
2020-05-04 21:04:51 +01:00
death-clawandGitHub 918f8578ad Update CHANGELOG.md 2020-05-03 16:40:26 +01:00
death-claw 10fa9f68ae v2.11.5 2020-05-03 15:18:28 +01:00
death-claw 6b5c9ccb0f Reduce rows buffer size 2020-05-03 15:16:31 +01:00
death-claw 652e8f1a9d v2.11.4 2020-05-03 14:24:27 +01:00
death-claw fe53ae733d Code cleanup
Dependencies update
2020-05-03 14:22:40 +01:00
death-claw 690dab2ae8 v2.11.3 2020-04-30 18:26:42 +01:00
death-claw 530542163e Add missing files 2020-04-30 18:26:23 +01:00
death-claw cf7f4c75c0 Source code restructure
UI fixes and enhancements
Add menu entry to rename a gallery to first alternative title
Add menu on toolbar for massive rename to first alternative title
2020-04-30 18:23:11 +01:00
death-claw c6b395cf4c v2.11.2 2020-04-26 22:06:34 +01:00
death-claw e72edeed62 Fix icon
Bug fixes
2020-04-26 22:02:51 +01:00
death-claw 3f22d53f89 v2.11.1 2020-04-25 16:32:25 +01:00
death-claw 08c0fa62b1 Add option to rename gallery
Code source maintenance
2020-04-25 16:28:56 +01:00
death-claw 4c79cca079 v2.11.0 2020-03-29 17:37:28 +01:00
death-clawandGitHub 67096242a2 Merge pull request #4 from life-claw/pixroute
Added Support for pixroute.com
2020-03-29 17:30:34 +01:00
death-claw f66112375d Merge branch 'master' of https://github.com/death-claw/vripper-project 2020-03-29 17:28:39 +01:00
death-claw c5a082e248 Upgrade Electron to v8
Upgrade Angular to v9
App settings are now portable in a json file
Add grab post title from thread option
General code source maintenance
2020-03-29 17:03:14 +01:00
life-claw c2a1b0aa33 Added Support for pixroute.com 2020-03-08 13:25:57 -05:00
death-clawandGitHub a89dcc96d8 Update README.md 2020-02-11 21:16:37 +01:00
death-claw b2a10eafd9 v2.10.8 2020-01-25 16:00:05 +01:00
death-claw f027aff89d Fix app url 2020-01-25 15:57:45 +01:00
death-claw b4eaf0d1c4 v2.10.7 2020-01-25 14:45:54 +01:00
death-claw 9d2975faaa Drop SockJs and switch to WebSocket
Autoreconnect front to back
2020-01-25 14:39:50 +01:00
death-claw 0c0fdf9d78 v2.10.6 2020-01-19 13:01:03 +01:00
death-claw e17afa0ef3 Limit cache size 2020-01-19 12:59:47 +01:00
death-claw 3d3eb6aa36 v2.10.5 2020-01-18 22:28:27 +01:00
death-claw e22132255e Bug fixes 2020-01-18 22:24:53 +01:00
death-claw ef041526ec v2.10.4 2020-01-18 13:27:47 +01:00
death-claw 704311dd9a Zero value for total max download will disable the limit
Add system notification
Bug fixes
2020-01-18 13:21:46 +01:00
death-claw 9617a6f20f v2.10.3 2020-01-16 21:39:59 +01:00
death-claw bc17442961 Some bug fixes
Enhance download queue logic
Clear cache button
Fix the partial status
2020-01-16 21:34:09 +01:00
death-claw 574d6818c1 Refactor PostImgHost 2020-01-14 21:03:35 +01:00
death-clawandGitHub 5c416055ee Merge pull request #3 from life-claw/postimg
Added Support for postimg.cc
2020-01-14 19:01:41 +01:00
death-clawandGitHub fa30458dc6 Merge pull request #1 from life-claw/imagevenue
Add support for imagevenue.com
2020-01-14 18:47:47 +01:00
death-clawandGitHub 638af93225 Merge pull request #2 from life-claw/urlSpaces
Sanitize URLs that contain spaces
2020-01-14 18:14:47 +01:00
life-claw 2c6ca741c7 Added PostImgHost 2020-01-07 16:50:11 -06:00
life-claw d7f2df2540 Sanitize URLs with that contain spaces 2020-01-07 16:18:45 -06:00
life-claw b85ade1dba Added ImageVenueHost 2020-01-07 16:04:04 -06:00
death-claw 1c61a0b34e v2.10.2 2019-12-29 11:35:59 +01:00
death-claw 46b9449969 Upgrade electron to v7
Update the retry policy
2019-12-29 11:33:03 +01:00
death-claw cc1bbd1231 v2.10.1 2019-12-27 23:05:10 +01:00
death-claw b29cba2911 Bug fix 2019-12-27 22:58:33 +01:00
death-claw 13bde3aa2c 2.10.0 2019-12-27 21:27:27 +01:00
death-claw 35b4f5fc97 Add support for imgspice
Fix spring dependency bug
2019-12-27 21:25:33 +01:00
death-claw 1b0482b7d4 v2.9.1 2019-12-27 20:10:31 +01:00
death-claw c9ff6eeab6 Fix build issue 2019-12-27 20:09:16 +01:00
death-claw 5ac57b743d v2.9.0 2019-12-27 20:07:19 +01:00
death-claw f7b219701f Allow many hosts to run at the same time
Add title to gallery
Add global concurrent downloads
Decrease timeout for fast fail
Add counter badge in link collector
2019-12-27 20:03:35 +01:00
death-claw d009aa3f8c v2.8.1 2019-12-24 20:21:29 +01:00
death-claw 078d49f883 v2.8.1 2019-12-24 20:18:22 +01:00
death-claw f1bf517ba5 Fix issue with path creation 2019-12-24 20:15:03 +01:00
death-claw 6269ed6f6e v2.8.0 2019-12-24 19:47:17 +01:00
death-claw 43b8dedbb1 Use pointer cursor on gallery items 2019-12-24 19:42:45 +01:00
death-claw 978be7bd49 Add view photos settings
Some improvements
2019-12-23 22:46:46 +01:00
death-claw 1b7f3aef62 Add missing css and script includes 2019-12-23 19:36:07 +01:00
death-claw 7d24ee4867 Use photoswipe for image gallery 2019-12-23 00:02:13 +01:00
death-claw 269a3475b0 Generate thumbnails on the fly and cache them 2019-12-21 19:17:51 +01:00
death-claw 18e2bc6afc Add Photo Viewer 2019-12-21 17:23:22 +01:00
death-claw c6ef54e115 v2.7.1 2019-12-21 11:24:03 +01:00
death-claw 441b2831fc Add version to title 2019-12-21 11:16:53 +01:00
death-claw 3cf4d60eca v2.7.0 2019-12-21 09:59:17 +01:00
death-claw d228c89a1c Add support for pixxxels
Add missing icon
2019-12-21 09:56:57 +01:00
death-claw f3b592e81b v2.6.0 2019-12-21 08:39:04 +01:00
death-claw a5ff659fb4 Better quality app icon
Implement context menu on download tab
Add stop remove and start button to context menu
Link will not be removed automatically from link collector tab
Some UI fixes
2019-12-21 08:34:56 +01:00
death-claw ab80e2f3b3 v2.5.1 2019-12-09 20:26:52 +01:00
death-claw 2ef082f87e Add hosts column in Link collector grab menu
Disable download button when nothing selected in Link collector grab menu
2019-12-09 19:56:26 +01:00
death-claw a1b97e76a7 v2.5.0 2019-12-08 20:13:58 +01:00
death-claw 185a078b4d Missing change 2019-12-08 20:08:42 +01:00
death-claw 28a4c944ee Add support for multiple links at a time
Add the Link collector feature
Fix bug with previews
2019-12-08 20:07:13 +01:00
death-claw 23d3553d99 Include startup icon
Fix UI layout
2019-12-06 23:09:57 +01:00
death-claw fb09e4c46b v2.4.1 2019-11-26 20:03:10 +01:00
death-claw 4feb6f5fe1 Increase attempts for establishing ws connection between front and back end
Change default download folder to Home folder (installer), App folder (portable)
2019-11-26 20:01:08 +01:00
death-claw df1b9efbbb v2.4.0 2019-11-23 14:50:16 +01:00
death-claw 553924fa1d Pimpandhost Host support 2019-11-23 14:48:05 +01:00
death-claw a90dbd568f v2.3.2 2019-11-23 12:15:42 +01:00
death-claw ca7ca1ce8e Fix a bug with the scan component 2019-11-23 12:14:07 +01:00
death-claw b3ab239a1e v2.3.1 2019-11-22 20:07:41 +01:00
death-claw 6e212c791c Fix connexion reset issue
Fix Post Status when autostart option is selected
Add stop all button
2019-11-22 20:05:14 +01:00
death-claw c6067ef147 v2.3.0 2019-11-21 21:17:25 +01:00
death-claw 8333f0c34e Fix Restoring post status on App startup
Use local sockjs and material icons font
2019-11-21 21:16:24 +01:00
death-claw 75becd8bb4 UI enhancements and overhaul
Stability fixes with HTTP connections
MacOS official support
2019-11-21 20:54:44 +01:00
death-claw 667376301a v2.2.5 2019-11-09 13:26:54 +01:00
death-claw 128e6b68a3 More robust parsing
Fix pixhost and imagezilla to exclude thumbs
2019-11-09 13:16:09 +01:00
death-claw a7c2844ab4 Change Retry duration 2019-11-09 10:13:12 +01:00
death-claw bd0b7d274b Fix macOs paths 2019-10-06 13:33:29 +01:00
death-claw bc57fad52b Remove failing rule 2019-10-06 12:00:01 +01:00
death-claw b81ef53cb9 Add support for macOs 2019-10-06 11:22:48 +01:00
death-claw c182bf0a08 v2.2.4 2019-10-05 18:06:40 +01:00
death-claw 07802b4b13 Add AppImage target 2019-10-05 18:03:32 +01:00
death-claw 50c4d947ed v2.2.3 2019-10-01 23:28:12 +01:00
death-claw b9b20e86dd Strip libjvm.so for debian 2019-10-01 23:27:01 +01:00
death-claw 64b93ce7e1 v2.2.2 2019-10-01 22:59:39 +01:00
death-claw 0bde4399e3 Update product name 2019-10-01 22:57:13 +01:00
death-claw 9b7818c5a7 v2.2.1 2019-10-01 22:44:19 +01:00
death-claw 1f2dcff353 Update dependency list for deb package 2019-10-01 22:42:28 +01:00
338 changed files with 20608 additions and 13323 deletions
+2 -1
View File
@@ -3,4 +3,5 @@
/**/*/target
/**/*/build-dir
/**/*/java-runtime
.idea
.idea
.vripper
+308 -1
View File
@@ -1,5 +1,312 @@
# Changelog
## [3.3.1] - 2021-02-18
### Added
- Add about menu
### Changed
- Fix README.md
## [3.3.0] - 2021-02-16
### Changed
- Add support for proxies
- Add a Event Logs
- Fix bugs with the database
## [3.2.3] - 2021-01-13
### Changed
- Finished posts are not cleared from the UI
## [3.2.2] - 2021-01-13
### Changed
- Set total download to max value
## [3.2.1] - 2021-01-12
### Added
- Add settings for connection timeout
- Add settings for maximum attempts
### Changed
- Enhance posts rename logic
- Dependencies update
- Switch reactive from rx-java to reactor
## [3.2.0] - 2020-11-16
### Changed
- Fix build issue
## [3.1.0] - 2020-11-16
### Changed
- UI revamp
- Fix bugs
## [3.0.11] - 2020-10-31
### Added
- Support for vipr.im
## [3.0.10] - 2020-10-04
### Changed
- Fix bug on settings update
## [3.0.9] - 2020-09-27
### Changed
- Revert to appending post id to download folders
- Fix a bug that lead to adding duplicate posts
- Remove old unused settings
## [3.0.8] - 2020-08-22
### Changed
- Bug fix which breaks concurrent downloads
## [3.0.7] - 2020-08-22
### Changed
- Fix MacOs build
## [3.0.6] - 2020-08-18
### Changed
- Fix TurboImageHost
## [3.0.5] - 2020-08-17
### Changed
- Update packages
- Host class refactoring
## [3.0.4] - 2020-08-16
### Changed
- Improve rename UI
## [3.0.3] - 2020-08-16
### Changed
- Fix bugs with electron app
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
- Introducing a proper database for persistence
## [2.12.1] - 2020-05-22
### Changed
- Fix alternative name bug
## [2.12.0] - 2020-05-21
### Changed
- Use new user hash from the api
## [2.11.7] - 2020-05-08
### Changed
- Fix bug with paths creation
- Move Open Link button
- Rename rename all title
## [2.11.6] - 2020-05-04
### Changed
- PNG filenames are now correctly named
- Fix issue with special characters in post titles
## [2.11.5] - 2020-05-03
### Changed
- Reduce rows buffer size
## [2.11.4] - 2020-05-03
### Changed
- Code cleanup
- Dependencies update
## [2.11.3] - 2020-04-30
### Changed
- Source code restructure
- UI fixes and enhancements
### Added
- Add menu entry to rename a gallery to first alternative title
- Add menu on toolbar for massive rename to first alternative title
## [2.11.2] - 2020-04-26
### Changed
- Bug fixes
## [2.11.1] - 2020-04-25
### Changed
- Code source maintenance
### Added
- Added option to rename gallery
## [2.11.0] - 2020-03-29
### Changed
- Upgrade Electron to v8
- Upgrade Angular to v9
- App settings are now portable in a json file
- General code source maintenance
### Added
- Added Support for pixroute.com
- Added grab post title from thread option
## [2.10.8] - 2020-01-25
### Changed
- Fix app url
## [2.10.7] - 2020-01-25
### Changed
- Drop SockJs and switch to WebSocket
- Autoreconnect front to back
## [2.10.6] - 2020-01-19
### Changed
- Limit cache size
## [2.10.5] - 2020-01-18
### Changed
- Bug fixes
## [2.10.4] - 2020-01-18
### Changed
- Zero value for total max download will disable the limit
- Add system notification
- Bug fixes
## [2.10.3] - 2020-01-16
### Changed
- Some bug fixes
- Enhance download queue logic
- Clear cache button
- Fix the partial status
- Sanitize URLs with spaces
### Added
- Support for Postimg host
- Support for Imagevenue host
## [2.10.2] - 2019-12-29
### Changed
- Upgrade electron to v7
- Bug fix
## [2.10.1] - 2019-12-27
### Changed
- Bug fix
## [2.10.0] - 2019-12-27
### Changed
- Fix spring dependency bug
### Added
- Add support for imgspice
## [2.9.0] - 2019-12-27
### Changed
- Fix build issue
## [2.9.0] - 2019-12-27
### Changed
- Decrease timeout for fast fail
### Added
- Allow many hosts to run at the same time
- Add title to gallery
- Add global concurrent downloads
- Add counter badge in link collector
## [2.8.1] - 2019-12-24
### Changed
- Fix issue with path creation
## [2.8.0] - 2019-12-24
### Added
- Photo gallery feature
## [2.7.1] - 2019-12-21
### Added
- Add version to title bar
## [2.7.0] - 2019-12-21
### Changed
- Add missing icon
### Added
- Add support for pixxxels
## [2.6.0] - 2019-12-21
### Changed
- Disable download button when nothing selected in Link collector grab menu
- Better quality app icon
- Implement context menu on download tab
- Add stop remove and start button to context menu
- Link will not be removed automatically from link collector tab
- Some UI fixes
## [2.5.1] - 2019-12-09
### Changed
- Disable download button when nothing selected in Link collector grab menu
### Added
- Add hosts column in Link collector grab menu
## [2.5.0] - 2019-12-08
### Changed
- Change loading screen
- UI Fixes
- Fix bug with previews
### Added
- Add link collector feature
- Add support for multiple links at a time
## [2.4.1] - 2019-11-26
### Changed
- Increase attempts for establishing ws connection between front and back end
- Change default download folder to Home folder (installer), App folder (portable)
## [2.4.0] - 2019-11-23
### Changed
- Pimpandhost Host support
## [2.3.2] - 2019-11-23
### Changed
- Fix a bug with the scan component
## [2.3.1] - 2019-11-22
### Changed
- Fix connexion reset issue
- Fix Post Status when autostart option is selected
- Add stop all button
## [2.3.0] - 2019-11-21
### Changed
- UI enhancements and overhaul
- Stability fixes with HTTP connections
- MacOS official support
- Use local sockjs and material icons font
## [2.2.5] - 2019-11-09
### Changed
- Exclude thumbs download for imagezilla and pixhost
- Change retry on fail logic (may help avoid having download errors)
## [2.2.4] - 2019-10-05
### Added
- AppImage target for linux
## [2.2.3] - 2019-10-01
### Changed
- Strip libjvm.so for debian
## [2.2.2] - 2019-10-01
### Changed
- Update product name
## [2.2.1] - 2019-10-01
### Changed
- Update dependency list for deb package
## [2.2.0] - 2019-10-01
### Added
- Embedd JRE within the app
@@ -166,4 +473,4 @@
- Add real time communication for progress updates via websocket (SockJS)
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
+14 -10
View File
@@ -1,26 +1,30 @@
# Viper Ripper!
# VRipper!
This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website.
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
## How to build
You need a recent version of maven 3.6.1+.
The application support 2 build modes:
- Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
- Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
To build the server app:
mvn clean install
mvn clean install -DskipTests
To build the desktop app:
mvn clean install -Pelectron
mvn clean install -Pelectron -DskipTests
Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
Maven will automatically handle front end compilation. However, for development, you will need to install a recent
version of nodejs on your system.
*Partially cross platform:
Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.
If you feel generous and want to support me, you can donate some coins, and I would be very grateful 🙏
I accept Bitcoin (BTC) for donations, following is my address
bc1q9jzzw3st49r9vmxgtylq4vrd67cl7zfclf3lsq
+3 -3
View File
@@ -4,12 +4,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>2.2.0</version>
<version>3.3.2</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.1.RELEASE</version>
<version>2.4.1</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<modules>
@@ -17,4 +17,4 @@
<module>vripper-ui</module>
<module>vripper-electron</module>
</modules>
</project>
</project>
Binary file not shown.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 303 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 592 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

+142 -113
View File
@@ -1,13 +1,16 @@
const { app, BrowserWindow } = require("electron");
require('v8-compile-cache');
const {app, BrowserWindow} = require("electron");
const path = require("path");
const url = require("url");
const getPort = require("get-port");
const { spawn } = require("child_process");
const { ipcMain } = require("electron");
const commandExists = require("command-exists").sync;
const { dialog } = require("electron");
const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const appDir = process.env.APPDIR;
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
const appImagePath = process.env.APPIMAGE;
let win;
let vripperServer;
@@ -19,124 +22,150 @@ let terminationInteval;
let terminated = false;
process.on("uncaughtException", err => {
dialog.showErrorBox(err.message, err.stack);
process.exit(1);
dialog.showErrorBox(err.message, err.stack);
process.exit(1);
});
if (!commandExists("java")) {
dialog.showErrorBox(
"Java command is missing",
"Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
);
process.exit(1);
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId("tn.mnlr.vripper");
}
let icon;
if (process.platform === "win32") {
icon = __dirname + '/icon.ico';
} else {
icon = __dirname + '/icon.png';
}
win = new BrowserWindow({
width: 1024,
height: 800,
minWidth: 800,
minHeight: 600,
webPreferences: {
nodeIntegration: true,
enableRemoteModule: true
},
icon: icon
});
win.removeMenu();
win.setMenu(null);
win.setMenuBarVisibility(false);
win.loadURL(
url.format({
pathname: path.join(__dirname, `vripper-ui/index.html`),
protocol: "file:",
slashes: true
})
);
// win.webContents.openDevTools();
win.on("closed", () => {
win = null;
});
}
function createWindow() {
let icon;
if(process.platform === "win32") {
icon = __dirname + '/icon.ico';
} else {
icon = __dirname + '/icon.png';
}
win = new BrowserWindow({
width: 1024,
height: 800,
minWidth: 800,
minHeight: 600,
webPreferences: {
nodeIntegration: true
},
icon: icon
});
win.removeMenu();
win.setMenu(null);
win.setMenuBarVisibility(false);
win.loadURL(
url.format({
pathname: path.join(__dirname, `vripper-ui/index.html`),
protocol: "file:",
slashes: true
})
);
// win.webContents.openDevTools();
win.on("closed", () => {
win = null;
});
shutdownServer = () => {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: {'content-type': 'application/json'},
}).then((response) => {
terminationInteval = setInterval(() => {
terminationAttemps++;
if (terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if (terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
}).catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
getPort().then(port => {
serverPort = port;
ipcMain.on("get-port", event => {
event.reply("port", port);
});
vripperServer = spawn(appDir !== undefined ? path.join(appDir, "java-runtime/bin/java") :
path.join(app.getPath('exe'), "../java-runtime/bin/java"), [
"-Xms256m",
"-Xmx1024m",
"-Dvripper.server.port=" + port,
"-jar",
appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
path.join(app.getPath('exe'), "../bin/vripper-server.jar")
], {
stdio: 'ignore'
});
vripperServer.on('exit', (code, signal) => {
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
terminated = true;
app.quit();
});
});
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
app.quit();
app.quit();
} else {
app.on("second-instance", (event, commandLine, workingDirectory) => {
if (win) {
if (win.isMinimized()) win.restore();
win.focus();
}
});
getPort().then(port => {
serverPort = port;
ipcMain.on("get-port", event => {
event.reply("port", port);
});
app.on("ready", createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: { 'content-type': 'application/json' },
}).then((response) => {
terminationInteval= setInterval(() => {
terminationAttemps++;
if(terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
const appPath = path.join(app.getAppPath(), '../../');
let javaBinPath, jarPath, baseDir;
if (appImageDir !== undefined && process.platform === 'linux') {
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
baseDir = path.join(appImagePath, '..');
} else if (process.platform === 'darwin') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = app.getPath('appData');
} else if (process.platform === 'win32') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = appPath;
} else {
console.error(`Unknown platform ${process.platform}`);
app.quit();
} else if(terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
})
.catch((error) => {
// Terminate immediately
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
});
}
app.on("activate", () => {
if (win === null) {
createWindow();
}
});
vripperServer = spawn(javaBinPath, [
"-Xms256m",
"-Dvripper.server.port=" + port,
"-Dbase.dir=" + baseDir,
"-jar",
jarPath
], {
stdio: 'ignore'
});
vripperServer.on('exit', (code, signal) => {
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
terminated = true;
app.quit();
});
});
app.on("second-instance", (event, commandLine, workingDirectory) => {
if (win) {
if (win.isMinimized()) win.restore();
win.focus();
}
});
app.on("ready", createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
shutdownServer();
}
});
app.on("will-quit", () => {
if (process.platform === "darwin") {
shutdownServer();
}
});
app.on("activate", () => {
if (win === null) {
createWindow();
}
});
}
+1094 -1695
View File
File diff suppressed because it is too large Load Diff
+20 -30
View File
@@ -1,6 +1,6 @@
{
"name": "vripper-electron",
"version": "2.2.0",
"version": "3.3.2",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "death-claw <53543762+death-claw@users.noreply.github.com>",
@@ -8,7 +8,7 @@
"license": "ISC",
"build": {
"appId": "tn.mnlr.vripper",
"productName": "V Ripper",
"productName": "VRipper",
"files": [
"**/*",
"!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -38,35 +38,26 @@
}
],
"win": {
"icon": "icon.png",
"icon": "icon.ico",
"target": [
"nsis"
"dir"
]
},
"nsis": {
"createDesktopShortcut": "always",
"oneClick": false,
"perMachine": false
},
"linux": {
"synopsis": "vipergirls.to ripper",
"category": "Utility",
"packageCategory": "Utility",
"icon": "icons",
"target": [
"deb"
"AppImage"
]
},
"deb": {
"depends": [
"gconf2",
"gconf-service",
"libnotify4",
"libappindicator1",
"libxtst6",
"libnss3",
"openjdk-8-jre"
]
"mac": {
"category": "public.app-category.utilities",
"target": [
"dmg"
],
"icon": "icon.icns"
}
},
"scripts": {
@@ -74,17 +65,16 @@
"dist": "node pre-build.js && electron-builder"
},
"devDependencies": {
"electron": "^6.0.9",
"electron-builder": "^21.2.0"
"electron": "11.1.1",
"electron-builder": "22.9.1"
},
"dependencies": {
"axios": "^0.19.0",
"cheerio": "^1.0.0-rc.3",
"command-exists": "^1.2.8",
"copy-dir": "^1.1.0",
"custom-electron-titlebar": "^3.0.10",
"electron-context-menu": "^0.13.0",
"get-port": "^5.0.0",
"rimraf": "^2.6.3"
"axios": "0.21.1",
"cheerio": "1.0.0-rc.3",
"copy-dir": "1.3.0",
"electron-context-menu": "2.1.0",
"get-port": "5.1.1",
"rimraf": "3.0.2",
"v8-compile-cache": "2.1.1"
}
}
+71 -71
View File
@@ -1,77 +1,77 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>2.2.0</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<classifier>${buildClassifier}</classifier>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>3.3.2</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.3</version>
<configuration>
<nodeVersion>v12.2.0</nodeVersion>
<npmVersion>6.9.0</npmVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<arguments>run dist</arguments>
<classifier>${buildClassifier}</classifier>
</configuration>
<phase>generate-resources</phase>
</execution>
</executions>
</plugin>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.3</version>
<configuration>
<nodeVersion>v12.2.0</nodeVersion>
<npmVersion>6.9.0</npmVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>run dist</arguments>
</configuration>
<phase>generate-resources</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
+6 -2
View File
@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
const fs = require('fs');
const rimraf = require("rimraf");
const copydir = require("copy-dir");
const { execSync } = require("child_process");
const {execSync} = require("child_process");
rimraf.sync('./build/vripper-ui');
copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -18,4 +18,8 @@ fs.writeFileSync('./build/vripper-ui/index.html', $.html());
console.log('Building runtime environment');
rimraf.sync('java-runtime');
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.naming,java.prefs,java.rmi,java.scripting,java.security.jgss,java.sql,jdk.httpserver,jdk.unsupported,jdk.crypto.ec --output java-runtime');
if (process.platform === 'linux') {
console.log('Stripping libjvm.so');
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
}
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
+22 -4
View File
@@ -5,7 +5,7 @@
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>2.2.0</version>
<version>3.3.2</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
@@ -32,6 +32,20 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.5.1</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -58,15 +72,19 @@
<version>2.22</version>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
<artifactId>rxjava</artifactId>
<version>2.2.6</version>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>net.jodah</groupId>
<artifactId>failsafe</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>28.1-jre</version>
</dependency>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-ui</artifactId>
@@ -0,0 +1,33 @@
package tn.mnlr.vripper;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.jpa.repositories.IRepository;
import tn.mnlr.vripper.services.DataService;
import java.util.Set;
@Component
public class EventListenerBean {
@Getter
private static boolean init = false;
private final DataService dataService;
private final Set<IRepository> repositorySet;
@Autowired
public EventListenerBean(DataService dataService, Set<IRepository> repositorySet) {
this.dataService = dataService;
this.repositorySet = repositorySet;
}
@EventListener
public void onApplicationEvent(ContextRefreshedEvent event) {
dataService.setDownloadingToStopped();
init = true;
}
}
@@ -1,7 +1,6 @@
package tn.mnlr.vripper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@@ -11,30 +10,24 @@ import org.springframework.stereotype.Component;
import java.util.Map;
@Component
@Slf4j
public class SpringContext implements ApplicationContextAware {
private static final Logger logger = LoggerFactory.getLogger(SpringContext.class);
private static ConfigurableApplicationContext context;
/**
* Returns the Spring managed bean instance of the given class type (if it exists).
* Returns null otherwise.
*
* @param beanClass
* @return
*/
public static <T extends Object> T getBean(Class<T> beanClass) {
public static <T> T getBean(Class<T> beanClass) {
return context.getBean(beanClass);
}
public static <T extends Object> Map<String, T> getBeansOfType(Class<T> beanClass) {
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
return context.getBeansOfType(beanClass);
}
public static void close() {
logger.info("Application terminating...");
context.close();
log.info("Application terminating...");
if (context != null) {
context.close();
}
}
@@ -0,0 +1,23 @@
package tn.mnlr.vripper;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
@Slf4j
public class Utils {
public static String throwableToString(Throwable th) {
try (StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter)) {
th.printStackTrace(printWriter);
printWriter.flush();
stringWriter.flush();
return stringWriter.toString();
} catch (IOException e) {
log.error("Failed to convert throwable to string", e);
}
return "Cannot display error, please check the log file for details";
}
}
@@ -1,81 +1,20 @@
package tn.mnlr.vripper;
import lombok.Getter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.exception.VripperException;
import tn.mnlr.vripper.services.AppSettingsService;
import tn.mnlr.vripper.services.PersistenceService;
import tn.mnlr.vripper.services.VipergirlsAuthService;
import javax.annotation.PostConstruct;
import java.io.File;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@SpringBootApplication
@Slf4j
public class VripperApplication {
private static final Logger logger = LoggerFactory.getLogger(VripperApplication.class);
// public static final String dataPath = System.getProperty("user.home", ".") + File.separator + ".vripper" + File.separator + "data.json";
public static final ExecutorService commonExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
public static void main(String[] args) {
try {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
SpringApplication.run(VripperApplication.class, args);
} catch (Exception e) {
logger.error("Failed to run the application", e);
}
}
@Component
public class AppCommandRunner implements CommandLineRunner {
@Autowired
private VipergirlsAuthService authService;
@Autowired
private PersistenceService persistenceService;
@Autowired
private AppSettingsService appSettingsService;
@Value("${base.dir}")
private String baseDir;
@Getter
private String dataPath;
@PostConstruct
public void init() {
dataPath = baseDir + File.separator + ".vripper" + File.separator + "data.json";
}
@Override
public void run(String... args) {
persistenceService.restore();
appSettingsService.restore();
registerShutdownHook();
try {
authService.authenticate();
} catch (VripperException e) {
logger.error("Cannot authenticate user with ViperGirls", e);
}
}
private void registerShutdownHook() {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
log.error("Failed to run the application", e);
}
}
}
@@ -0,0 +1,277 @@
package tn.mnlr.vripper.download;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.function.CheckedRunnable;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.*;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
@Slf4j
public class DownloadJob implements CheckedRunnable {
private static final Object LOCK = new Object();
private static final int READ_BUFFER_SIZE = 8192;
private final DataService dataService;
private final PathService pathService;
private final ConnectionService cm;
private final VGAuthService authService;
private final DownloadSpeedService downloadSpeedService;
private final SettingsService settingsService;
private final HttpClientContext context;
@Getter
private final Image image;
@Getter
private final Post post;
private volatile boolean stopped = false;
@Getter
private boolean finished = false;
DownloadJob(Post post, Image image) {
this.image = image;
this.post = post;
dataService = SpringContext.getBean(DataService.class);
pathService = SpringContext.getBean(PathService.class);
cm = SpringContext.getBean(ConnectionService.class);
authService = SpringContext.getBean(VGAuthService.class);
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
settingsService = SpringContext.getBean(SettingsService.class);
context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
context.setAttribute(ContextAttributes.OPEN_CONNECTION.toString(), Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
}
public void download(final Post post, final Image image) throws DownloadException {
try {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING) && !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
// The post may be updated and the download directory might be set by another thread
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
if (updatedPost.getDownloadDirectory() == null) {
pathService.createDefaultPostFolder(updatedPost);
}
if (settingsService.getSettings().getLeaveThanksOnStart()) {
authService.leaveThanks(updatedPost);
}
}
if (stopped) {
return;
}
/*
* HOST SPECIFIC
*/
log.debug(String.format("Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image.getUrl(), context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
log.debug(String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
/*
* END HOST SPECIFIC
*/
if (stopped) {
return;
}
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
log.debug(String.format("Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
HttpClient client = cm.getClient().build();
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
httpGet.addHeader("Referer", image.getUrl());
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (stopped) {
return;
}
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
try (InputStream downloadStream = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(outputFile)) {
if (stopped) {
return;
}
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
if (stopped) {
return;
}
}
checkImageTypeAndRename(dataService.findById(post.getId()).orElseThrow(), outputFile, nameAndUrl.getName(), image.getIndex());
}
} catch (Exception e) {
if (stopped) {
return;
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
image.setStatus(Status.COMPLETE);
} else if (stopped) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
finished = true;
}
}
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index) throws HostException {
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
imageName += ".jpg";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_jpg")) {
toReplace = "_jpg";
} else if (imageNameLC.endsWith("_jpeg")) {
toReplace = "_jpeg";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
}
}
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_png")) {
imageName += ".png";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_png")) {
toReplace = "_png";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
}
}
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
pathService.getDirectoryAccess().lock();
File downloadDestinationFolder = pathService.calcDownloadDirectory(post);
File outImage = new File(downloadDestinationFolder, (settingsService.getSettings().getForceOrder() ? String.format("%03d_", index) : "") + imageName);
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
} finally {
pathService.getDirectoryAccess().unlock();
try {
Files.delete(outputFile.toPath());
} catch (IOException e) {
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
}
}
}
@Override
public void run() throws Exception {
if (stopped) {
finished = true;
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
download(post, image);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return Objects.equals(image, that.image) &&
Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image, post);
}
public void stop() {
List<AbstractExecutionAwareRequest> requests = (List<AbstractExecutionAwareRequest>) this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
this.stopped = true;
}
public enum ContextAttributes {
OPEN_CONNECTION("OPEN_CONNECTION");
private final String value;
ContextAttributes(String value) {
this.value = value;
}
@Override
public String toString() {
return value;
}
}
}
@@ -0,0 +1,54 @@
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import net.jodah.failsafe.Failsafe;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.Utils;
import tn.mnlr.vripper.jpa.domain.Event;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.DataService;
import java.time.LocalDateTime;
@Slf4j
public class DownloadRunnable implements Runnable {
private final DownloadService downloadService;
private final DataService dataService;
private final ConnectionService connectionService;
private final IEventRepository eventRepository;
private final DownloadJob downloadJob;
public DownloadRunnable(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
connectionService = SpringContext.getBean(ConnectionService.class);
eventRepository = SpringContext.getBean(IEventRepository.class);
this.downloadJob = downloadJob;
}
@Override
public void run() {
Failsafe.with(connectionService.getRetryPolicy())
.onFailure(e -> {
try {
Event event = new Event(Event.Type.DOWNLOAD, Event.Status.ERROR, LocalDateTime.now(), String.format("Failed to download %s\n %s", downloadJob.getImage().getUrl(), Utils.throwableToString(e.getFailure())));
eventRepository.save(event);
} catch (Exception exp) {
log.error("Failed to save event", exp);
}
log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
})
.onComplete(e -> {
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
downloadService.afterJobFinish(downloadJob);
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
}).run(downloadJob);
}
}
@@ -0,0 +1,223 @@
package tn.mnlr.vripper.download;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.PostService;
import tn.mnlr.vripper.services.SettingsService;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@Slf4j
public class DownloadService {
private static final List<Status> FINISHED = Arrays.asList(Status.ERROR, Status.COMPLETE, Status.STOPPED);
private final int MAX_POOL_SIZE = 12;
private final ConcurrentHashMap<Host, AtomicInteger> threadCount = new ConcurrentHashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final BlockingQueue<DownloadJob> executionQueue = new LinkedBlockingQueue<>();
private final List<DownloadJob> executing = Collections.synchronizedList(new ArrayList<>());
private final PendingQueue pendingQueue;
private final SettingsService settings;
private final DataService dataService;
private final PostService postService;
private boolean pauseQ = false;
private Thread executionThread;
private Thread pollThread;
@Autowired
public DownloadService(PendingQueue pendingQueue, SettingsService settings, DataService dataService, PostService postService) {
this.pendingQueue = pendingQueue;
this.settings = settings;
this.dataService = dataService;
this.postService = postService;
}
@PostConstruct
private void init() {
executionThread = new Thread(this::start, "Executor thread");
pollThread = new Thread(this::poll, "Polling thread");
pollThread.start();
executionThread.start();
}
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
executionThread.interrupt();
pollThread.interrupt();
executor.shutdown();
dataService.findAllPosts().forEach(p -> {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
executor.awaitTermination(5, TimeUnit.SECONDS);
}
private void stopRunning(@NonNull String postId) {
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = executing.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::stop);
} else {
dataService.findAllPosts().forEach(p -> this.stop(p.getPostId()));
}
}
public void restartAll(List<String> posIds) {
if (posIds != null) {
posIds.forEach(this::restart);
} else {
dataService.findAllPosts().forEach(p -> this.restart(p.getPostId()));
}
}
private void restart(@NonNull String postId) {
if (isPending(postId)) {
log.warn(String.format("Cannot restart, jobs are currently running for post id %s", postId));
return;
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
return;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
post.setStatus(Status.PENDING);
dataService.updatePostStatus(post.getStatus(), post.getId());
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postId));
for (Image image : images) {
try {
pendingQueue.put(post, image);
} catch (InterruptedException e) {
log.warn("Thread was interrupted", e);
Thread.currentThread().interrupt();
}
}
}
private boolean isPending(String postId) {
return pendingQueue.isPending(postId);
}
private void stop(String postId) {
try {
pauseQ = true;
final Post post = dataService.findPostByPostId(postId).orElseThrow();
if (post == null) {
return;
}
if (FINISHED.contains(post.getStatus())) {
return;
}
pendingQueue.stop(post);
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
postService.stopFetchingMetadata(post);
} finally {
pauseQ = false;
}
}
private boolean canRun(Host host) {
boolean canRun;
AtomicInteger count = threadCount.get(host);
if (count == null) {
threadCount.put(host, new AtomicInteger(0));
}
canRun = threadCount.get(host).get() < settings.getSettings().getMaxThreads() && (settings.getSettings().getMaxTotalThreads() == 0 ? threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < MAX_POOL_SIZE : threadCount.values().stream().mapToInt(AtomicInteger::get).sum() < settings.getSettings().getMaxTotalThreads());
if (canRun && !pauseQ) {
threadCount.get(host).incrementAndGet();
return true;
}
return false;
}
private void poll() {
while (!Thread.interrupted()) {
try {
List<DownloadJob> peek = pendingQueue.peek();
for (DownloadJob downloadJob : peek) {
if (canRun(downloadJob.getImage().getHost())) {
executionQueue.offer(downloadJob);
pendingQueue.remove(downloadJob);
}
}
synchronized (threadCount) {
threadCount.wait(2_000);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
private void start() {
while (!Thread.interrupted()) {
try {
push(executionQueue.take());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
} catch (Exception e) {
log.error("Execution Service failed", e);
break;
}
}
}
private void push(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new DownloadRunnable(downloadJob));
executing.add(downloadJob);
}
public synchronized void afterJobFinish(DownloadJob downloadJob) {
int count = pendingQueue.decrement(downloadJob.getPost().getPostId());
if (count == 0) {
dataService.finishPost(downloadJob.getPost());
}
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
executing.remove(downloadJob);
synchronized (threadCount) {
threadCount.notify();
}
}
public int runningCount() {
return executing.size();
}
}
@@ -0,0 +1,115 @@
package tn.mnlr.vripper.download;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.services.DataService;
import tn.mnlr.vripper.services.SettingsService;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
@Service
@Slf4j
public class PendingQueue {
private final DataService dataService;
private final SettingsService settingsService;
private final List<Host> hosts;
private final ConcurrentHashMap<Host, BlockingDeque<DownloadJob>> pendingQ = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> toBeExecuted = new ConcurrentHashMap<>();
@Autowired
public PendingQueue(DataService dataService, SettingsService settingsService, List<Host> hosts) {
this.dataService = dataService;
this.settingsService = settingsService;
this.hosts = hosts;
}
@PostConstruct
private void init() {
hosts.forEach(host -> pendingQ.put(host, new LinkedBlockingDeque<>()));
}
public void put(Post post, Image image) throws InterruptedException {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob = new DownloadJob(post, image);
pendingQ.get(downloadJob.getImage().getHost()).putLast(downloadJob);
checkKey(post.getPostId());
toBeExecuted.get(post.getPostId()).incrementAndGet();
}
private synchronized void checkKey(String postId) {
if (!toBeExecuted.containsKey(postId)) {
toBeExecuted.put(postId, new AtomicInteger(0));
}
}
public void remove(final DownloadJob downloadJob) {
pendingQ.get(downloadJob.getImage().getHost()).remove(downloadJob);
}
public List<DownloadJob> peek() {
List<DownloadJob> downloadJobs = new ArrayList<>();
if (hosts.size() == 0) {
return downloadJobs;
}
for (Host host : hosts) {
Iterator<DownloadJob> it = pendingQ.get(host).iterator();
for (int i = 0; i < settingsService.getSettings().getMaxThreads(); i++) {
DownloadJob downloadJob = it.hasNext() ? it.next() : null;
if (downloadJob != null) {
downloadJobs.add(downloadJob);
}
}
}
return downloadJobs;
}
public void enqueue(Post post, Set<Image> images) throws InterruptedException {
for (Image image : images) {
put(post, image);
}
}
public int size() {
return toBeExecuted.values().stream().mapToInt(AtomicInteger::get).sum();
}
public void stop(Post post) {
Predicate<DownloadJob> predicate = next -> next.getImage().getPostId().equals(post.getPostId());
for (Map.Entry<Host, BlockingDeque<DownloadJob>> entry : pendingQ.entrySet()) {
entry.getValue().stream().filter(predicate).forEach(e -> decrement(post.getPostId()));
entry.getValue().removeIf(predicate);
decrement(post.getPostId());
}
}
public boolean isPending(String postId) {
return toBeExecuted.containsKey(postId);
}
public synchronized int decrement(String postId) {
AtomicInteger counter = toBeExecuted.get(postId);
if (counter == null) {
return 0;
}
int count = counter.decrementAndGet();
if (count == 0) {
toBeExecuted.remove(postId);
}
return count;
}
}
@@ -1,134 +0,0 @@
package tn.mnlr.vripper.entities;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
@Getter
@NoArgsConstructor
public class Image {
private static final Logger logger = LoggerFactory.getLogger(Image.class);
private AppStateService appStateService;
private Disposable subscription;
private final String type = "img";
private String postId;
private String postName;
private Host host;
private String url;
private int index;
private AtomicLong current = new AtomicLong(0);
private Status status;
private BehaviorProcessor<Image> imageStateProcessor;
@Setter
private long total = 0;
public Image(String url, String postId, String postName, Host host, int index) {
this.url = url;
this.postId = postId;
this.postName = postName;
this.host = host;
this.index = index;
status = Status.PENDING;
imageStateProcessor = BehaviorProcessor.create();
}
public void setAppStateService(AppStateService appStateService) {
this.appStateService = appStateService;
appStateService.getCurrentImages().put(this.url, this);
appStateService.getLiveImageUpdates().onNext(this);
}
public void setStatus(Status status) {
this.status = status;
update();
}
public boolean isCompleted() {
return status.equals(Status.COMPLETE);
}
public void init() {
if (appStateService == null) {
logger.warn("Attempting to init the image whilst the App State is null, unexpected behaviour could occur from this");
return;
}
cleanup();
imageStateProcessor = BehaviorProcessor.create();
subscription = imageStateProcessor
.onBackpressureLatest()
.doOnNext(appStateService::onImageUpdate)
.subscribe();
current.set(0);
status = Status.PENDING;
imageStateProcessor.onNext(this);
}
public void cleanup() {
if (imageStateProcessor != null) {
imageStateProcessor.onComplete();
}
if (subscription != null) {
subscription.dispose();
}
}
public void setCurrent(int current) {
this.current.set(current);
update();
}
public void increase(int read) {
current.addAndGet(read);
update();
}
private void update() {
if(imageStateProcessor == null) {
return;
}
imageStateProcessor.onNext(this);
if (isCompleted()) {
imageStateProcessor.onComplete();
subscription.dispose();
imageStateProcessor = null;
}
}
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return url.equals(image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -1,119 +0,0 @@
package tn.mnlr.vripper.entities;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Getter
@ToString
@NoArgsConstructor
public class Post {
private AppStateService appStateService;
private Status status;
private final String type = "post";
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private List<Image> images;
private Map<String, Object> metadata;
private AtomicInteger done = new AtomicInteger(0);
private int total;
private Set<String> hosts;
private boolean removed = false;
private String forum;
public Post(String title, String url, List<Image> images, Map<String, Object> metadata, String postId, String threadId, String threadTitle, String forum) {
this.title = title;
this.url = url;
this.images = images;
this.metadata = metadata;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
if (this.images.contains(null)) {
System.out.println("Oops");
}
this.images.stream().map(e -> {
if (e == null) {
System.out.println("Am the cause");
}
return e.getHost();
}).collect(Collectors.toSet());
this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
this.hosts = this.images.stream().map(e -> e.getHost()).map(Host::getHost).collect(Collectors.toSet());
total = images.size();
status = Status.PENDING;
}
public void setAppStateService(AppStateService appStateService) {
this.appStateService = appStateService;
this.appStateService.getCurrentPosts().put(postId, this);
this.appStateService.getLivePostsState().onNext(this);
}
public void setRemoved(boolean removed) {
this.removed = removed;
updateNotification();
}
public void increase() {
done.incrementAndGet();
updateNotification();
}
public void setStatus(Status status) {
this.status = status;
updateNotification();
}
private void updateNotification() {
if (appStateService != null) {
appStateService.getLivePostsState().onNext(this);
}
}
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, PARTIAL, STOPPED
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return postId.equals(post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -1,12 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.databind.util.StdConverter;
import tn.mnlr.vripper.host.Host;
public class HostToString extends StdConverter<Host, String> {
@Override
public String convert(Host value) {
return value.getClass().getSimpleName();
}
}
@@ -1,32 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
public abstract class ImagePersistanceMixin {
@JsonSerialize(converter = HostToString.class)
@JsonDeserialize(converter = StringToHost.class)
private Host host;
@JsonIgnore
private BehaviorProcessor<Image> imageStateProcessor;
@JsonIgnore
private Disposable subscription;
@JsonIgnore
private String type;
@JsonIgnore
public abstract boolean isCompleted();
@JsonIgnore
private AppStateService appStateService;
}
@@ -1,17 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.annotation.JsonIgnore;
import tn.mnlr.vripper.services.AppStateService;
public abstract class PostPersistanceMixin {
@JsonIgnore
private String type;
@JsonIgnore
private AppStateService appStateService;
@JsonIgnore
private boolean removed;
}
@@ -1,17 +0,0 @@
package tn.mnlr.vripper.entities.mixin.persistance;
import com.fasterxml.jackson.databind.util.StdConverter;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import java.util.Collection;
public class StringToHost extends StdConverter<String, Host> {
private Collection<Host> hosts = SpringContext.getBeansOfType(Host.class).values();
@Override
public Host convert(String value) {
return hosts.stream().filter(e -> e.getClass().getSimpleName().equals(value)).findAny().orElse(null);
}
}
@@ -1,23 +0,0 @@
package tn.mnlr.vripper.entities.mixin.ui;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.reactivex.disposables.Disposable;
import io.reactivex.processors.BehaviorProcessor;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.services.AppStateService;
public abstract class ImageUIMixin {
@JsonIgnore
private Host host;
@JsonIgnore
private BehaviorProcessor<Image> imageStateProcessor;
@JsonIgnore
private Disposable subscription;
@JsonIgnore
private AppStateService appStateService;
}
@@ -1,16 +0,0 @@
package tn.mnlr.vripper.entities.mixin.ui;
import com.fasterxml.jackson.annotation.JsonIgnore;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.services.AppStateService;
import java.util.List;
public abstract class PostUIMixin {
@JsonIgnore
private List<Image> images;
@JsonIgnore
private AppStateService appStateService;
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class EventRemoveEvent extends ApplicationEvent {
private final Long id;
public EventRemoveEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class EventUpdateEvent extends ApplicationEvent {
private final Long id;
public EventUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class ImageUpdateEvent extends ApplicationEvent {
private final Long id;
public ImageUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class MetadataUpdateEvent extends ApplicationEvent {
private final Long postIdRef;
public MetadataUpdateEvent(Object source, Long postIdRef) {
super(source);
this.postIdRef = postIdRef;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class PostRemoveEvent extends ApplicationEvent {
private final String postId;
public PostRemoveEvent(Object source, String postId) {
super(source);
this.postId = postId;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class PostUpdateEvent extends ApplicationEvent {
private final Long id;
public PostUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class QueuedRemoveEvent extends ApplicationEvent {
private final String threadId;
public QueuedRemoveEvent(Object source, String threadId) {
super(source);
this.threadId = threadId;
}
}
@@ -0,0 +1,15 @@
package tn.mnlr.vripper.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
@Getter
public class QueuedUpdateEvent extends ApplicationEvent {
private final Long id;
public QueuedUpdateEvent(Object source, Long id) {
super(source);
this.id = id;
}
}
@@ -6,7 +6,7 @@ public class DownloadException extends Exception {
super(message);
}
public DownloadException(Exception e) {
public DownloadException(Throwable e) {
super(e);
}
}
@@ -1,7 +1,7 @@
package tn.mnlr.vripper.exception;
public class HostException extends Exception {
public HostException(Exception e) {
public HostException(Throwable e) {
super(e);
}
@@ -9,7 +9,7 @@ public class HostException extends Exception {
super(message);
}
public HostException(String message, Exception e) {
super(message,e);
public HostException(String message, Throwable e) {
super(message, e);
}
}
@@ -2,7 +2,7 @@ package tn.mnlr.vripper.exception;
public class HtmlProcessorException extends Exception {
public HtmlProcessorException(Exception e) {
public HtmlProcessorException(Throwable e) {
super(e);
}
}
@@ -6,11 +6,11 @@ public class PostParseException extends Exception {
super(message);
}
public PostParseException(String message, Exception e) {
public PostParseException(String message, Throwable e) {
super(message, e);
}
public PostParseException(Exception e) {
public PostParseException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,16 @@
package tn.mnlr.vripper.exception;
public class QueueException extends Exception {
public QueueException(String message) {
super(message);
}
public QueueException(String message, Throwable e) {
super(message, e);
}
public QueueException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,10 @@
package tn.mnlr.vripper.exception;
import java.io.IOException;
public class RenameException extends Exception {
public RenameException(String message, IOException e) {
super(message, e);
}
}
@@ -5,7 +5,7 @@ public class VripperException extends Exception {
super(message);
}
public VripperException(Exception e) {
public VripperException(Throwable e) {
super(e);
}
}
@@ -1,7 +1,7 @@
package tn.mnlr.vripper.exception;
public class XpathException extends Exception {
public XpathException(Exception e) {
public XpathException(Throwable e) {
super(e);
}
}
@@ -1,37 +1,49 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class AcidimgHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(AcidimgHost.class);
private static final String host = "acidimg.cc";
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
public static final String IMG_XPATH = "//img[@class='centred']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final ConnectionService cm;
private final HostService hostService;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public AcidimgHost(ConnectionService cm, HostService hostService, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.hostService = hostService;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -39,22 +51,27 @@ public class AcidimgHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node contDiv;
try {
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.info(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
HttpPost httpPost = cm.buildHttpPost(url, context);
httpPost.addHeader("Referer", url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
@@ -64,9 +81,9 @@ public class AcidimgHost extends Host {
throw new HostException(e);
}
logger.info(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
logger.info(String.format("Cleaning response for %s", httpPost));
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
@@ -76,23 +93,22 @@ public class AcidimgHost extends Host {
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if(imgNode == null) {
if (imgNode == null) {
throw new HostException("Cannot find the image node");
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? getDefaultImageName(imgUrl) : imgTitle);
return new HostService.NameUrl(imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -1,231 +1,38 @@
package tn.mnlr.vripper.host;
import lombok.Getter;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import tn.mnlr.vripper.entities.Image;
import tn.mnlr.vripper.exception.DownloadException;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.*;
import tn.mnlr.vripper.services.HostService;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Iterator;
import java.util.Random;
import java.util.Objects;
@Service
@Slf4j
abstract public class Host {
private static final Logger logger = LoggerFactory.getLogger(Host.class);
private static final int READ_BUFFER_SIZE = 8192;
@Autowired
protected HtmlProcessorService htmlProcessorService;
@Autowired
protected XpathService xpathService;
@Autowired
private AppSettingsService appSettingsService;
@Autowired
private AppStateService appStateService;
@Autowired
private ConnectionManager cm;
@Autowired
private DownloadSpeedService downloadSpeedService;
@Autowired
private PathService pathService;
abstract public String getHost();
abstract public String getLookup();
public boolean isSupported(String url) {
return url.contains(getHost());
return url.contains(getLookup());
}
public void download(Image image, ImageFileData imageFileData) throws DownloadException, InterruptedException {
public abstract HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException;
try {
appStateService.postDownloadingUpdate(image.getPostId());
imageFileData.setPageUrl(image.getUrl());
/**
* HOST SPECIFIC
*/
logger.info(String.format("Getting image url and name from %s using %s", image.getUrl(), this.getHost()));
setNameAndUrl(image.getUrl(), imageFileData);
logger.info(String.format("Resolved name for %s: %s", image.getUrl(), imageFileData.getImageName()));
logger.info(String.format("Resolved image url for %s: %s", image.getUrl(), imageFileData.getImageUrl()));
logger.info(String.format("Building image request for %s", image.getUrl()));
setImageRequest(imageFileData);
/**
* END HOST SPECIFIC
*/
String formatImageFileName = pathService.formatImageFileName(imageFileData.getImageName());
logger.info(String.format("Sanitizing image name from %s to %s", imageFileData.getImageName(), formatImageFileName));
imageFileData.setImageName(formatImageFileName);
File destinationFolder = pathService.getDownloadDestinationFolder(image.getPostId());
logger.info(String.format("Saving to %s", destinationFolder.getPath()));
if (!destinationFolder.exists()) {
logger.info(String.format("Creating %s", destinationFolder.getPath()));
destinationFolder.mkdirs();
}
HttpClient client = cm.getClient().build();
logger.info(String.format("Downloading %s", imageFileData.getImageUrl()));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(imageFileData.getImageRequest())) {
if(response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
File outputFile = new File(destinationFolder.getPath() + File.separator + imageFileData.getImageName() + ".tmp");
InputStream downloadStream = response.getEntity().getContent();
FileOutputStream fos = new FileOutputStream(outputFile);
try {
image.setTotal(response.getEntity().getContentLength());
logger.info(String.format("%s length is %d", imageFileData.getImageUrl(), image.getTotal()));
logger.info(String.format("Starting data transfer for %s", imageFileData.getImageUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
}
EntityUtils.consumeQuietly(response.getEntity());
} finally {
if (downloadStream != null) {
downloadStream.close();
}
if (fos != null) {
fos.close();
}
}
checkImageTypeAndRename(outputFile, imageFileData.getImageName(), image.getIndex());
}
} catch (Exception e) {
if(Thread.interrupted()) {
throw new InterruptedException("Download was interrupted");
}
throw new DownloadException(e);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Host host = (Host) o;
return Objects.equals(getHost(), host.getHost());
}
private void checkImageTypeAndRename(File outputFile, String imageName, int index) throws HostException {
String formatName;
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
formatName = reader.getFormatName();
if (formatName.toUpperCase().equals("JPEG")) {
formatName = "jpg";
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
File outImage = new File(outputFile.getParent(), (appSettingsService.isForceOrder() ? String.format("%03d_", index) : "") + imageName + "." + formatName.toLowerCase());
if (outImage.exists()) {
outImage.delete();
}
Files.move(outputFile.toPath(), outImage.toPath());
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
}
}
/**
* Just for testing, you may ignore
* @throws Exception
*/
private void randomFail() throws Exception {
Random random = new Random();
int i = random.nextInt(100);
if (i < 50) {
throw new Exception("Purpose error");
}
}
protected final String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
logger.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
protected final Response getResponse(final String url) throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
Header[] headers;
logger.info(String.format("Requesting %s", url));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpGet)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(String.format("Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
logger.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
protected void setImageRequest(final ImageFileData imageFileData) {
HttpGet httpGet = cm.buildHttpGet(imageFileData.getImageUrl());
httpGet.addHeader("Referer", imageFileData.getPageUrl());
imageFileData.setImageRequest(httpGet);
}
protected abstract void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException;
@Getter
public static class Response {
protected Response(Document document, Header[] headers) {
this.document = document;
this.headers = headers;
}
private Document document;
private Header[] headers;
@Override
public int hashCode() {
return Objects.hash(getHost());
}
@Override
@@ -1,12 +1,11 @@
package tn.mnlr.vripper.host;
import org.apache.http.Header;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -14,24 +13,33 @@ import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.io.IOException;
import java.util.Arrays;
import java.util.stream.Collectors;
@Service
@Slf4j
public class ImageBamHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageBamHost.class);
private static final String host = "imagebam.com";
public static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
public static final String IMG_XPATH = "//img[@class='image']";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to your image']";
private static final String IMG_XPATH = "//img[@class='image']";
private final HostService hostService;
private final ConnectionService cm;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public ImageBamHost(HostService hostService, ConnectionService cm, XpathService xpathService, HtmlProcessorService htmlProcessorService) {
this.hostService = hostService;
this.cm = cm;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -39,37 +47,34 @@ public class ImageBamHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Response response = getResponse(url);
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
HostService.Response response = hostService.getResponse(url, context);
Document doc = response.getDocument();
Header[] headers = response.getHeaders();
String cookies = Arrays.asList(headers)
.stream()
.filter(e -> e.getName().toLowerCase().contains("Set-Cookie".toLowerCase()))
.map(e -> e.getValue().split(";")[0].trim())
.collect(Collectors.joining("; "));
Node contDiv;
try {
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
logger.info(String.format("Getting cookies to use for %s", url));
logger.info(String.format("Click button found for %s", url));
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url);
HttpGet httpGet = cm.buildHttpGet(url, context);
httpGet.addHeader("Referer", url);
httpGet.addHeader("Cookie", cookies);
logger.info(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet)) {
log.debug(String.format("Requesting %s", httpGet));
try (CloseableHttpResponse res = (CloseableHttpResponse) client.execute(httpGet, context)) {
String s = EntityUtils.toString(res.getEntity());
logger.debug(String.format("%s response is:%n%s", httpGet, s));
logger.debug(String.format("Cleaning response for %s", httpGet));
log.debug(String.format("%s response is:%n%s", httpGet, s));
log.debug(String.format("Cleaning response for %s", httpGet));
doc = htmlProcessorService.clean(s);
EntityUtils.consumeQuietly(res.getEntity());
} catch (IOException | HtmlProcessorException e) {
@@ -79,20 +84,19 @@ public class ImageBamHost extends Host {
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("id").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
} catch(Exception e) {
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
@@ -1,26 +1,33 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class ImageTwistHost extends Host {
public static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final Logger logger = LoggerFactory.getLogger(ImageTwistHost.class);
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "imagetwist.com";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public ImageTwistHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,25 +35,29 @@ public class ImageTwistHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,82 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImageVenueHost extends Host {
private static final String host = "imagevenue.com";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageVenueHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
//Button detected. No need to actually click it, just make the call again.
resp = hostService.getResponse(url, context);
doc = resp.getDocument();
}
} catch (XpathException e) {
throw new HostException(e);
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,26 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImageZillaHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImageZillaHost.class);
private static final String host = "imagezilla.net";
public static final String IMG_XPATH = "//img[@id='photo']";
private static final String lookup = "imagezilla.net/show";
private static final String IMG_XPATH = "//img[@id='photo']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public ImageZillaHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,16 +34,21 @@ public class ImageZillaHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return lookup;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
String title;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
logger.info(String.format("Resolving name for %s", url));
if(titleNode != null) {
log.debug(String.format("Resolving name for %s", url));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
@@ -46,13 +57,12 @@ public class ImageZillaHost extends Host {
throw new HostException(e);
}
if(title == null || title.isEmpty()) {
title = getDefaultImageName(url);
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(url);
}
try {
imageFileData.setImageUrl(url.replace("show", "images"));
imageFileData.setImageName(title);
return new HostService.NameUrl(title, url.replace("show", "images"));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,65 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImgSpiceHost extends Host {
private static final String host = "imgspice.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,26 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class ImgboxHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImgboxHost.class);
private static final String host = "imgbox.com";
public static final String IMG_XPATH = "//img[@id='img']";
private static final String IMG_XPATH = "//img[@id='img']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public ImgboxHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,25 +34,29 @@ public class ImgboxHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle);
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -1,14 +1,14 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@@ -16,24 +16,35 @@ import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.HtmlProcessorException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.ConnectionService;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.HtmlProcessorService;
import tn.mnlr.vripper.services.XpathService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@Service
@Slf4j
public class ImxHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(ImxHost.class);
private static final String host = "imx.to";
public static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
public static final String IMG_XPATH = "//img[@class='centred']";
private static final String CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final HostService hostService;
private final XpathService xpathService;
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
@Autowired
private ConnectionManager cm;
public ImxHost(HostService hostService, XpathService xpathService, ConnectionService cm, HtmlProcessorService htmlProcessorService) {
this.hostService = hostService;
this.xpathService = xpathService;
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
@@ -41,17 +52,25 @@ public class ImxHost extends Host {
}
@Override
protected void setNameAndUrl(final String _url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Response resp = getResponse(url);
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node contDiv;
String value = null;
try {
logger.info(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
if (contDiv == null) {
throw new HostException(CONTINUE_BUTTON_XPATH + " cannot be found");
}
Node node = contDiv.getAttributes().getNamedItem("value");
if (node != null) {
value = node.getTextContent();
@@ -60,46 +79,43 @@ public class ImxHost extends Host {
throw new HostException(e);
}
if (contDiv != null) {
if (value == null) {
throw new HostException("Failed to obtain value attribute from continue input");
}
logger.info(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", value));
try {
httpPost.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
throw new HostException(e);
}
logger.info(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost)) {
logger.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
throw new HostException(e);
}
if (value == null) {
throw new HostException("Failed to obtain value attribute from continue input");
}
log.debug(String.format("Click button found for %s", url));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(url, context);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", value));
try {
httpPost.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
throw new HostException(e);
}
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response = (CloseableHttpResponse) client.execute(httpPost, context)) {
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException | HtmlProcessorException e) {
throw new HostException(e);
}
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle);
} catch(Exception e) {
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
@@ -0,0 +1,75 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PimpandhostHost extends Host {
private static final String host = "pimpandhost.com";
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PimpandhostHost(HostService hostService, XpathService xpathService) {
super();
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url;
try {
url = hostService.appendUri(_url.replace("http://", "https://"), "size=original");
} catch (Exception e) {
throw new HostException(e);
}
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = "https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle,
imgUrl
);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,69 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PixRouteHost extends Host {
private static final String host = "pixroute.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixRouteHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
return new HostService.NameUrl(
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
imgNode.getAttributes().getNamedItem("src").getTextContent().trim()
);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,26 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PixhostHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(PixhostHost.class);
private static final String host = "pixhost.to";
public static final String IMG_XPATH = "//img[@id='image']";
private static final String lookup = "pixhost.to/show";
private static final String IMG_XPATH = "//img[@id='image']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public PixhostHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -28,25 +34,29 @@ public class PixhostHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return lookup;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
logger.info(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
logger.info(String.format("Resolving name and image url for %s", url));
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
imageFileData.setImageUrl(imgUrl);
imageFileData.setImageName(imgTitle.substring(imgTitle.indexOf('_') + 1));
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class PixxxelsHost extends Host {
private static final String host = "pixxxels.cc";
private static final String IMG_XPATH = "//*[@id='download']";
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixxxelsHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = titleNode.getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
return new HostService.NameUrl(imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,69 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class PostImgHost extends Host {
private static final String host = "postimg.cc";
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
private static final String IMG_XPATH = "//a[@id='download']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PostImgHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String _url, final HttpClientContext context) throws HostException {
String url = _url.replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node urlNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(titleNode).map(node -> node.getTextContent().trim()).orElseGet(() -> hostService.getDefaultImageName(url));
return new HostService.NameUrl(imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -1,27 +1,32 @@
package tn.mnlr.vripper.host;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.q.ImageFileData;
import tn.mnlr.vripper.services.ConnectionManager;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
@Service
@Slf4j
public class TurboImageHost extends Host {
private static final Logger logger = LoggerFactory.getLogger(TurboImageHost.class);
private static final String host = "turboimagehost.com";
public static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
public static final String IMG_XPATH = "//img[@id='uImage']";
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
private static final String IMG_XPATH = "//img[@id='imageid']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
private ConnectionManager cm;
public TurboImageHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
@@ -29,16 +34,21 @@ public class TurboImageHost extends Host {
}
@Override
protected void setNameAndUrl(final String url, final ImageFileData imageFileData) throws HostException {
public String getLookup() {
return host;
}
Document doc = getResponse(url).getDocument();
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
String title;
try {
logger.info(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
logger.info(String.format("Resolving name for %s", url));
if(titleNode != null) {
log.debug(String.format("Resolving name for %s", url));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
@@ -47,14 +57,13 @@ public class TurboImageHost extends Host {
throw new HostException(e);
}
if(title == null || title.isEmpty()) {
title = getDefaultImageName(url);
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(url);
}
try {
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
imageFileData.setImageUrl(urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
imageFileData.setImageName(title);
return new HostService.NameUrl(title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
@@ -0,0 +1,65 @@
package tn.mnlr.vripper.host;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import tn.mnlr.vripper.exception.HostException;
import tn.mnlr.vripper.exception.XpathException;
import tn.mnlr.vripper.services.HostService;
import tn.mnlr.vripper.services.XpathService;
import java.util.Optional;
@Service
@Slf4j
public class ViprImHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "vipr.im";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ViprImHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final String url, final HttpClientContext context) throws HostException {
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt")).map(Node::getTextContent).map(String::trim).orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.jpa;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tn.mnlr.vripper.download.DownloadService;
import tn.mnlr.vripper.services.ThreadPoolService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
@EnableScheduling
public class Management {
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Pattern BACKUP_FILE_PATTERN = Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
private final JdbcTemplate jdbcTemplate;
private final String backupFolder;
private final ThreadPoolService threadPoolService;
private final DownloadService downloadService;
public Management(JdbcTemplate jdbcTemplate, @Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName, ThreadPoolService threadPoolService, DownloadService downloadService) {
this.jdbcTemplate = jdbcTemplate;
this.threadPoolService = threadPoolService;
this.downloadService = downloadService;
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
}
@PreDestroy
public void destroy() {
try {
threadPoolService.destroy();
downloadService.destroy();
this.jdbcTemplate.execute("SHUTDOWN");
} catch (Exception ignored) {
Thread.currentThread().interrupt();
}
}
@PostConstruct
@Scheduled(cron = "0 0 0 ? * *")
private void backup() {
for (File file : Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[]{})) {
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
if (matcher.find()) {
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
if (localDate.isEqual(LocalDate.now())) {
return;
}
}
}
// create a backup file
File backupFile = new File(backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
}
}
@@ -0,0 +1,64 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Event {
private Long id;
private Type type;
private Status status;
@JsonSerialize(using = DateTimeSerializer.class)
private LocalDateTime time;
private String message;
public Event(Type type, Status status, LocalDateTime time, String message) {
this.type = type;
this.status = status;
this.time = time;
this.message = message;
}
public enum Type {
POST,
QUEUED,
THANKS,
METADATA,
SCAN,
DOWNLOAD,
METADATA_CACHE_MISS,
QUEUED_CACHE_MISS
}
public enum Status {
PENDING, PROCESSING, DONE, ERROR
}
}
class DateTimeSerializer extends StdSerializer<LocalDateTime> {
private final static DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
protected DateTimeSerializer() {
super(LocalDateTime.class);
}
@Override
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider) throws IOException {
gen.writeString(value.format(DATE_TIME_FORMATTER));
}
}
@@ -0,0 +1,70 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Objects;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Image {
@JsonIgnore
protected Long id;
@JsonIgnore
private Host host;
private String url;
private int index;
private long current = 0;
private long total = 0;
private Status status;
private String postId;
@JsonIgnore
private Long postIdRef;
public Image(String postId, String url, Host host, int index) {
this.postId = postId;
this.url = url;
this.host = host;
this.index = index;
status = Status.STOPPED;
}
public void increase(int read) {
current += read;
}
public void init() {
current = 0;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return Objects.equals(url, image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -0,0 +1,22 @@
package tn.mnlr.vripper.jpa.domain;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.Collections;
import java.util.List;
@Getter
@Setter
@NoArgsConstructor
public class Metadata {
private Long postIdRef;
private String PostId;
private String postedBy;
private List<String> resolvedNames = Collections.emptyList();
}
@@ -0,0 +1,78 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Post {
@JsonIgnore
private Long id;
private Status status;
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private int done;
private int total;
private Set<String> hosts;
private String forum;
@JsonIgnore
private String securityToken;
@JsonIgnore
private String downloadDirectory;
private boolean thanked;
private Set<String> previews = Collections.emptySet();
private Metadata metadata;
public Post(String title, String url, String postId, String threadId, String threadTitle, String forum, String securityToken) {
this.title = title;
this.url = url;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return Objects.equals(postId, post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -0,0 +1,57 @@
package tn.mnlr.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import java.util.Objects;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class Queued {
@JsonIgnore
private Long id;
private String link;
private String threadId;
private String postId;
private int total = 0;
private boolean loading = true;
public Queued(String link, String threadId, String postId) {
this();
this.link = link;
this.threadId = threadId;
this.postId = postId;
}
public void done() {
this.loading = false;
}
public void increment() {
this.total++;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Queued that = (Queued) o;
return Objects.equals(threadId, that.threadId);
}
@Override
public int hashCode() {
return Objects.hash(threadId);
}
}
@@ -0,0 +1,5 @@
package tn.mnlr.vripper.jpa.domain.enums;
public enum Status {
PENDING, DOWNLOADING, COMPLETE, ERROR, STOPPED, PARTIAL
}
@@ -0,0 +1,21 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Event;
import java.util.List;
import java.util.Optional;
public interface IEventRepository extends IRepository {
Event save(Event event);
Event update(Event event);
List<Event> findAll();
Optional<Event> findById(Long id);
void delete(Long id);
void deleteAll();
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.List;
import java.util.Optional;
public interface IImageRepository extends IRepository {
Image save(Image image);
void deleteAllByPostId(String postId);
List<Image> findByPostId(String postId);
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
int stopByPostIdAndIsNotCompleted(String postId);
List<Image> findByPostIdAndIsError(String postId);
Optional<Image> findById(Long id);
int updateStatus(Status status, Long id);
int updateCurrent(long current, Long id);
int updateTotal(long total, Long id);
}
@@ -0,0 +1,14 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Metadata;
import java.util.Optional;
public interface IMetadataRepository extends IRepository {
Metadata save(Metadata metadata);
Optional<Metadata> findByPostId(String postId);
int deleteByPostId(String postId);
}
@@ -0,0 +1,36 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.util.List;
import java.util.Optional;
public interface IPostRepository extends IRepository {
Post save(Post post);
Optional<Post> findByPostId(String postId);
Optional<Post> findById(Long id);
List<String> findCompleted();
List<Post> findAll();
boolean existByPostId(String postId);
int setDownloadingToStopped();
int deleteByPostId(String postId);
int updateStatus(Status status, Long id);
int updateDone(int done, Long id);
int updateFolderName(String postFolderName, Long id);
int updateTitle(String title, Long id);
int updateThanked(boolean thanked, Long id);
}
@@ -0,0 +1,20 @@
package tn.mnlr.vripper.jpa.repositories;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.util.List;
import java.util.Optional;
public interface IQueuedRepository extends IRepository {
Queued save(Queued queued);
Optional<Queued> findByThreadId(String threadId);
List<Queued> findAll();
Optional<Queued> findById(Long id);
int deleteByThreadId(String threadId);
void deleteAll();
}
@@ -0,0 +1,4 @@
package tn.mnlr.vripper.jpa.repositories;
public interface IRepository {
}
@@ -0,0 +1,117 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.EventRemoveEvent;
import tn.mnlr.vripper.event.EventUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Event;
import tn.mnlr.vripper.jpa.repositories.IEventRepository;
import tn.mnlr.vripper.services.SettingsService;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Optional;
@Service
@Slf4j
public class EventRepository implements IEventRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private final SettingsService settingsService;
private ApplicationEventPublisher applicationEventPublisher;
public EventRepository(JdbcTemplate jdbcTemplate, SettingsService settingsService) {
this.jdbcTemplate = jdbcTemplate;
this.settingsService = settingsService;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject(
"CALL NEXT VALUE FOR SEQ_EVENT",
Long.class);
}
@Override
public synchronized Event save(@NonNull Event event) {
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
if (count > maxRecords) {
List<Long> idList = jdbcTemplate.queryForList("SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
idList.forEach(this::delete);
}
long id = nextId();
jdbcTemplate.update(
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
id,
event.getType().name(),
event.getStatus().name(),
event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
event.getMessage()
);
event.setId(id);
applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, id));
return event;
}
@Override
public Event update(@NonNull Event event) {
if (event.getId() == null) {
log.warn("Cannot update entity with null id");
return event;
}
jdbcTemplate.update(
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
event.getType().name(),
event.getStatus().name(),
event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
event.getMessage(),
event.getId()
);
applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, event.getId()));
return event;
}
@Override
public Optional<Event> findById(Long id) {
List<Event> events = jdbcTemplate.query(
"SELECT * FROM EVENT WHERE ID = ?",
new EventRowMapper(),
id);
if (events.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(events.get(0));
}
}
@Override
public List<Event> findAll() {
return jdbcTemplate.query(
"SELECT * FROM EVENT",
new EventRowMapper());
}
@Override
public void delete(Long id) {
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
applicationEventPublisher.publishEvent(new EventRemoveEvent(EventRepository.class, id));
}
@Override
public void deleteAll() {
jdbcTemplate.update("DELETE FROM EVENT");
}
@Override
public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,23 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Event;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
public class EventRowMapper implements RowMapper<Event> {
@Override
public Event mapRow(ResultSet rs, int rowNum) throws SQLException {
Event event = new Event();
event.setId(rs.getLong("ID"));
event.setType(Event.Type.valueOf(rs.getString("TYPE")));
event.setStatus(Event.Status.valueOf(rs.getString("STATUS")));
event.setTime(LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
event.setMessage(rs.getString("MESSAGE"));
return event;
}
}
@@ -0,0 +1,146 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.ImageUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IImageRepository;
import java.util.List;
import java.util.Optional;
@Service
public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public ImageRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject(
"CALL NEXT VALUE FOR SEQ_IMAGE",
Long.class);
}
@Override
public Image save(Image image) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
id,
image.getCurrent(),
image.getHost().getHost(),
image.getIndex(),
image.getPostId(),
image.getStatus().name(),
image.getTotal(),
image.getUrl(),
image.getPostIdRef()
);
image.setId(id);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return image;
}
@Override
public void deleteAllByPostId(String postId) {
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
}
@Override
public List<Image> findByPostId(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE WHERE POST_ID = ?",
new ImageRowMapper(),
postId);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'",
Integer.class
);
}
@Override
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
new ImageRowMapper(),
postId);
}
@Override
public int stopByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
postId
);
}
@Override
public List<Image> findByPostIdAndIsError(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
new ImageRowMapper(),
postId);
}
@Override
public Optional<Image> findById(Long id) {
List<Image> images = jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.ID = ?",
new ImageRowMapper(),
id);
if (images.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(images.get(0));
}
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?",
status.name(), id
);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return mutationCount;
}
@Override
public int updateCurrent(long current, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?",
current, id
);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return mutationCount;
}
@Override
public int updateTotal(long total, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?",
total, id
);
applicationEventPublisher.publishEvent(new ImageUpdateEvent(ImageRepository.class, id));
return mutationCount;
}
@Override
public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,29 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.SpringContext;
import tn.mnlr.vripper.host.Host;
import tn.mnlr.vripper.jpa.domain.Image;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.sql.ResultSet;
import java.sql.SQLException;
public class ImageRowMapper implements RowMapper<Image> {
@Override
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
Image image = new Image();
image.setId(rs.getLong("ID"));
String host = rs.getString("HOST");
image.setHost(SpringContext.getBeansOfType(Host.class).values().stream().filter(e -> e.getHost().equals(host)).findAny().orElse(null));
image.setUrl(rs.getString("URL"));
image.setIndex(rs.getInt("INDEX"));
image.setCurrent(rs.getLong("CURRENT"));
image.setTotal(rs.getLong("TOTAL"));
image.setStatus(Status.valueOf(rs.getString("STATUS")));
image.setPostId(rs.getString("POST_ID"));
image.setPostIdRef(rs.getLong("POST_ID_REF"));
return image;
}
}
@@ -0,0 +1,65 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.MetadataUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
import java.util.List;
import java.util.Optional;
@Service
public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public MetadataRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public Metadata save(Metadata metadata) {
jdbcTemplate.update(
"INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
metadata.getPostIdRef(),
metadata.getPostId(),
metadata.getPostedBy(),
String.join("%sep%", metadata.getResolvedNames())
);
applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, metadata.getPostIdRef()));
return metadata;
}
@Override
public Optional<Metadata> findByPostId(String postId) {
List<Metadata> metadata = jdbcTemplate.query(
"SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
new MetadataRowMapper(),
postId
);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
postId
);
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,24 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Metadata;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
public class MetadataRowMapper implements RowMapper<Metadata> {
@Override
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
Metadata metadata = new Metadata();
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
metadata.setPostId(rs.getString("POST_ID"));
metadata.setPostedBy(rs.getString("POSTED_BY"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
return metadata;
}
}
@@ -0,0 +1,188 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.PostRemoveEvent;
import tn.mnlr.vripper.event.PostUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import tn.mnlr.vripper.jpa.repositories.IPostRepository;
import java.util.List;
import java.util.Optional;
@Service
public class PostRepository implements IPostRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public PostRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject(
"CALL NEXT VALUE FOR SEQ_POST",
Long.class);
}
@Override
public Post save(Post post) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
String.join(";", post.getHosts()),
post.getDownloadDirectory(),
post.getPostId(),
String.join(";", post.getPreviews()),
post.getSecurityToken(),
post.getStatus().name(),
post.isThanked(),
post.getThreadId(),
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
post.getUrl()
);
post.setId(id);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return post;
}
@Override
public Optional<Post> findByPostId(String postId) {
List<Post> posts = jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
new PostRowMapper(),
postId);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<String> findCompleted() {
return jdbcTemplate.query(
"SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total",
((rs, rowNum) -> rs.getString("POST_ID"))
);
}
@Override
public Optional<Post> findById(Long id) {
List<Post> posts = jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
new PostRowMapper(),
id);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<Post> findAll() {
return jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
new PostRowMapper()
);
}
@Override
public boolean existByPostId(String postId) {
Integer count = jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?",
Integer.class,
postId);
if (count == null) {
return false;
} else {
return count > 0;
}
}
@Override
public int setDownloadingToStopped() {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'"
);
}
@Override
public int deleteByPostId(String postId) {
int mutationCount = jdbcTemplate.update(
"DELETE FROM POST AS post WHERE post.POST_ID = ?",
postId
);
applicationEventPublisher.publishEvent(new PostRemoveEvent(PostRepository.class, postId));
return mutationCount;
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?",
status.name(), id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateDone(int done, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?",
done, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateFolderName(String postFolderName, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
postFolderName, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateTitle(String title, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?",
title, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public int updateThanked(boolean thanked, Long id) {
int mutationCount = jdbcTemplate.update(
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?",
thanked, id
);
applicationEventPublisher.publishEvent(new PostUpdateEvent(PostRepository.class, id));
return mutationCount;
}
@Override
public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,55 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Metadata;
import tn.mnlr.vripper.jpa.domain.Post;
import tn.mnlr.vripper.jpa.domain.enums.Status;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Set;
public class PostRowMapper implements RowMapper<Post> {
private static final String DELIMITER = ";";
@Override
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
Post post = new Post();
post.setId(rs.getLong("post.ID"));
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
post.setPostId(rs.getString("post.POST_ID"));
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
post.setThreadId(rs.getString("post.THREAD_ID"));
post.setTitle(rs.getString("post.TITLE"));
post.setUrl(rs.getString("post.URL"));
post.setDone(rs.getInt("post.DONE"));
post.setTotal(rs.getInt("post.TOTAL"));
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
post.setForum(rs.getString("post.FORUM"));
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
post.setThanked(rs.getBoolean("post.THANKED"));
String previews;
if ((previews = rs.getString("post.PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
Long metadataId = rs.getLong("metadata.POST_ID_REF");
if (!rs.wasNull()) {
Metadata metadata = new Metadata();
metadata.setPostIdRef(metadataId);
metadata.setPostId(rs.getString("metadata.POST_ID"));
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
post.setMetadata(metadata);
}
return post;
}
}
@@ -0,0 +1,105 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import tn.mnlr.vripper.event.QueuedRemoveEvent;
import tn.mnlr.vripper.event.QueuedUpdateEvent;
import tn.mnlr.vripper.jpa.domain.Queued;
import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
import java.util.List;
import java.util.Optional;
@Service
public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware {
private final JdbcTemplate jdbcTemplate;
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
public QueuedRepository(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject(
"CALL NEXT VALUE FOR SEQ_QUEUED",
Long.class);
}
@Override
public Queued save(Queued queued) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
id,
queued.getTotal(),
queued.getLink(),
queued.isLoading(),
queued.getPostId(),
queued.getThreadId()
);
queued.setId(id);
applicationEventPublisher.publishEvent(new QueuedUpdateEvent(QueuedRepository.class, id));
return queued;
}
@Override
public Optional<Queued> findByThreadId(String threadId) {
List<Queued> queuedList = jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
new Object[]{threadId},
new QueuedRowMapper()
);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public List<Queued> findAll() {
return jdbcTemplate.query(
"SELECT * FROM QUEUED",
new QueuedRowMapper()
);
}
@Override
public Optional<Queued> findById(Long id) {
List<Queued> queuedList = jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?",
new Object[]{id},
new QueuedRowMapper()
);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public int deleteByThreadId(String threadId) {
int mutationCount = jdbcTemplate.update(
"DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?",
threadId
);
applicationEventPublisher.publishEvent(new QueuedRemoveEvent(QueuedRepository.class, threadId));
return mutationCount;
}
@Override
public void deleteAll() {
jdbcTemplate.update("DELETE FROM QUEUED");
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
}
}
@@ -0,0 +1,22 @@
package tn.mnlr.vripper.jpa.repositories.impl;
import org.springframework.jdbc.core.RowMapper;
import tn.mnlr.vripper.jpa.domain.Queued;
import java.sql.ResultSet;
import java.sql.SQLException;
public class QueuedRowMapper implements RowMapper<Queued> {
@Override
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
Queued queued = new Queued();
queued.setId(rs.getLong("ID"));
queued.setLink(rs.getString("LINK"));
queued.setThreadId(rs.getString("THREAD_ID"));
queued.setPostId(rs.getString("POST_ID"));
queued.setTotal(rs.getInt("TOTAL"));
queued.setLoading(rs.getBoolean("LOADING"));
return queued;
}
}
@@ -0,0 +1,7 @@
package tn.mnlr.vripper.listener;
import reactor.core.publisher.Flux;
public interface DataEventListener<T> {
Flux<T> getDataFlux();
}
@@ -0,0 +1,7 @@
package tn.mnlr.vripper.listener;
import reactor.core.publisher.Sinks.EmitFailureHandler;
public class EmitHandler {
public static final EmitFailureHandler RETRY = (signalType, emitResult) -> true;
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.EventRemoveEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class EventRemoveEventListener implements ApplicationListener<EventRemoveEvent>, DataEventListener<EventRemoveEvent> {
private final Sinks.Many<EventRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(EventRemoveEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<EventRemoveEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,33 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.EventUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class EventUpdateEventListener implements ApplicationListener<EventUpdateEvent>, DataEventListener<EventUpdateEvent> {
private final Sinks.Many<EventUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(@NonNull EventUpdateEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<EventUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.ImageUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class ImageUpdateEventListener implements ApplicationListener<ImageUpdateEvent>, DataEventListener<ImageUpdateEvent> {
private final Sinks.Many<ImageUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(ImageUpdateEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<ImageUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.MetadataUpdateEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class MetadataUpdateEventListener implements ApplicationListener<MetadataUpdateEvent>, DataEventListener<MetadataUpdateEvent> {
private final Sinks.Many<MetadataUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(MetadataUpdateEvent event) {
sink.emitNext(event, (signalType, emitResult) -> true);
}
@Override
public Flux<MetadataUpdateEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}
@@ -0,0 +1,32 @@
package tn.mnlr.vripper.listener;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
import tn.mnlr.vripper.event.PostRemoveEvent;
import javax.annotation.PreDestroy;
@Component
@Slf4j
public class PostRemoveEventListener implements ApplicationListener<PostRemoveEvent>, DataEventListener<PostRemoveEvent> {
private final Sinks.Many<PostRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
@Override
public void onApplicationEvent(PostRemoveEvent event) {
sink.emitNext(event, EmitHandler.RETRY);
}
@Override
public Flux<PostRemoveEvent> getDataFlux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
}
}

Some files were not shown because too many files have changed in this diff Show More