Compare commits

...
252 Commits
Author SHA1 Message Date
dev-claw a8e5dbf5f7 change package id 2025-09-29 17:50:02 +01:00
dev-claw 70cee5cb17 change artifact id 2025-09-29 17:46:48 +01:00
dev-claw 9012cc065e backport fixes for Imx and ImageBam 2025-09-29 17:37:50 +01:00
death-claw c018033d2f Code enhancements 2021-05-31 20:22:16 +01:00
death-claw b0f5a15943 Add support for dpic.me 2021-05-31 19:06:41 +01:00
death-claw 8db4d406ad v3.5.4 2021-05-29 10:32:26 +01:00
death-claw 47f1705abb Fix download queue bug 2021-05-29 10:30:18 +01:00
death-claw 816fb84549 v3.5.3 2021-05-24 15:17:30 +01:00
death-claw 5213ed7129 Fix icon 2021-05-24 15:15:49 +01:00
death-claw 2483c5fa00 v3.5.2 2021-05-24 14:59:32 +01:00
death-claw dfc5810fd3 Add missing icon 2021-05-24 14:57:47 +01:00
death-claw fad418134e Update donation infos 2021-05-24 14:56:00 +01:00
death-claw 9dd91bfc07 v3.5.1 2021-05-22 15:51:05 +01:00
death-claw 5bd42e2ce5 Merge remote-tracking branch 'origin/master' 2021-05-22 15:44:22 +01:00
death-claw e18832b154 v3.5.0 2021-05-21 20:11:16 +01:00
death-claw 41feb53a83 Bug fix 2021-05-20 23:46:59 +01:00
death-claw 6f166b42c4 Enhance metadata interruption
Add Added on and Order columns
2021-05-20 21:25:52 +01:00
death-claw e8ee579a37 Save window size and position
Resize confirmation box
2021-05-18 21:10:42 +01:00
death-claw 41ce99995b Download service enhancements 2021-05-10 21:19:40 +01:00
death-claw 5caeef0e70 v3.4.2 2021-05-10 11:18:43 +01:00
death-claw 96ef0cb234 Fix race condition 2021-05-10 11:17:02 +01:00
death-claw 641b3e3e7b v3.4.1 2021-05-10 00:21:23 +01:00
death-claw 08de4dc5b0 Fix icons 2021-05-10 00:19:42 +01:00
death-claw 57680a6053 v3.4.0 2021-05-09 22:45:54 +01:00
death-claw 53fd14935b Enhance download queue logic 2021-05-09 22:39:22 +01:00
death-claw 90af03d68f Fix imagebam
Code enhancements
2021-05-09 14:12:48 +01:00
death-claw 265614585b Parameterize settings 2021-05-09 10:35:41 +01:00
death-claw 4503d540ed Implement an event bus 2021-05-09 00:17:32 +01:00
death-claw b08b730dff replace guava with caffeine 2021-05-08 13:03:20 +01:00
death-claw 06d636329e v3.3.8 2021-05-05 18:31:13 +01:00
death-claw 85313600fc Fix fonts path 2021-05-05 18:29:50 +01:00
death-claw 7d451397f9 v3.3.7 2021-05-05 18:16:00 +01:00
death-claw 1c94b93597 Fix imagebam host and code format 2021-05-05 18:14:25 +01:00
death-claw 8f09596453 Fix electron build 2021-03-20 11:35:46 +01:00
death-claw 6ce05e78b3 v3.3.6 2021-03-20 11:28:20 +01:00
death-claw 67a375150c Optimize build size 2021-03-20 11:27:06 +01:00
death-claw fba5f670a0 v3.3.5 2021-03-18 09:31:22 +01:00
death-claw 7d083630cc Bug fix 2021-03-18 09:29:45 +01:00
death-claw 335cbf70b4 v3.3.4 2021-03-18 09:12:48 +01:00
death-claw e25d306f1d Bug fixes 2021-03-18 09:11:01 +01:00
death-claw f395338a9d Update build plugins 2021-03-13 17:08:13 +01:00
death-claw 58a40682cc v3.3.3 2021-03-13 16:21:36 +01:00
death-claw ccc4032bb0 Remove unnecessary event logs 2021-03-13 16:15:56 +01:00
death-claw 1c5b8a0397 v3.3.2 2021-03-11 19:05:09 +01:00
death-claw cc2ce9482d Fix a bug with event logs
Change event names and add some icons
2021-03-11 19:02:01 +01:00
death-claw 41f1775360 v3.3.1 2021-02-18 16:04:34 +01:00
death-claw 733f1a08ac Add about menu
Fix README.md
2021-02-18 16:01:06 +01:00
death-claw 0cb0ee0100 v3.3.0 2021-02-16 19:15:58 +01:00
death-claw cb88e39c83 Add support for proxies
Add a Event Logs
Fix bugs with the database
2021-02-16 19:13:02 +01:00
death-claw 6375c0aed6 v3.2.3 2021-01-13 20:15:13 +01:00
death-claw 40ca4cf86b Finished posts are not cleared from the UI 2021-01-13 20:11:47 +01:00
death-clawandGitHub b3a798388b Update README.md 2021-01-13 12:54:51 +01:00
death-claw 6e69b2bce8 v3.2.2 2021-01-13 09:51:56 +01:00
death-claw bb6dcd77b9 Set total download to max value 2021-01-13 09:49:38 +01:00
death-claw 42e1aa73e2 v3.2.1 2021-01-12 20:02:17 +01:00
death-claw 26fde22af2 Code reformat 2021-01-12 19:37:12 +01:00
death-claw 798f39d54f Add settings for connection timeout
Add settings for maximum attempts
Enhance posts rename logic
Dependencies update
Switch reactive from rx-java to reactor
2021-01-12 19:32:43 +01:00
death-claw 3a21dfac38 Remove unused font 2021-01-06 14:55:49 +01:00
death-claw e8355b88b9 v3.2 2020-11-16 19:40:36 +01:00
death-claw 379660236d Commit missing files 2020-11-16 19:31:59 +01:00
death-claw c86ee91c27 v3.1.0 2020-11-16 16:44:26 +01:00
death-claw ff9d253242 UI revamp
Fix bugs
2020-11-16 16:42:30 +01:00
death-claw 53b45e00ba v3.0.11 2020-10-31 15:09:19 +01:00
death-claw 247b12a525 Add support for vipr.im 2020-10-31 15:06:26 +01:00
death-claw 7a8d332e6e v3.0.10 2020-10-04 16:31:57 +01:00
death-claw 749a6f18d2 Fix bug on settings update 2020-10-04 16:27:26 +01:00
death-claw 4107048f12 v3.0.9 2020-09-27 10:14:08 +01:00
death-claw f5d9ab85a2 Revert to appending post id to download folders
Fix a bug that lead to adding duplicate posts
Remove old unused settings
2020-09-27 10:12:29 +01:00
death-claw b82ee7b494 v3.0.8 2020-08-22 19:01:08 +01:00
death-claw 8a31dfc012 Bug fix which breaks concurrent downloads 2020-08-22 18:59:44 +01:00
death-claw 734c1b72c7 v3.0.7 2020-08-22 16:45:43 +01:00
death-claw 7abb7df1f6 Fix MacOs build 2020-08-22 16:44:24 +01:00
death-claw 0a7fa99a64 v3.0.6 2020-08-18 08:33:57 +01:00
death-claw 691026c8a9 Fix TurboImageHost 2020-08-18 08:32:33 +01:00
death-claw 3ef004b947 v3.0.5 2020-08-17 20:06:12 +01:00
death-claw 1b22a93e28 Update packages
Host class refactoring
2020-08-17 20:04:39 +01:00
death-claw 2639df0aaa Update packages
Host class refactoring
2020-08-17 20:04:23 +01:00
death-claw d5ae5d57d1 v3.0.4 2020-08-16 16:11:11 +01:00
death-claw 537d76d7be Improve rename UI 2020-08-16 16:06:17 +01:00
death-claw c1247faacc v3.0.3 2020-08-16 11:29:01 +01:00
death-claw efcc685f30 Remove notification feature 2020-08-16 11:26:57 +01:00
death-claw c825b22ce4 Fix some bugs with MacOS 2020-08-16 10:58:19 +01:00
death-claw 6ef5d5474f Fix some bugs with MacOS 2020-08-16 10:20:56 +01:00
death-claw bd25801c6c Fix base dir configuration for MacOS 2020-08-16 00:18:18 +01:00
death-claw 377b93c1ac Fix base dir configuration for AppImage 2020-08-16 00:02:18 +01:00
death-claw d498d20f6b Set default download path to user's home 2020-08-15 23:23:43 +01:00
death-claw d18544d414 Fix bin location for mac 2020-08-15 23:10:58 +01:00
death-claw 3c872b79bd Update app data configuration 2020-08-15 23:00:48 +01:00
death-claw 37900cac49 v3.0.2 2020-08-15 15:38:53 +01:00
death-claw be57c76d93 Fix bugs with download queue 2020-08-15 15:36:47 +01:00
death-claw 8b52ad7858 v3.0.1 2020-08-03 18:48:45 +01:00
death-claw a0976bde02 Fix bugs with download queue 2020-08-03 18:43:42 +01:00
death-claw 15c97d5cfc v3.0.0 2020-08-03 15:06:36 +01:00
death-claw 905cbd5a35 Code refactoring
Fix bugs with rename gallery
2020-08-03 15:03:42 +01:00
death-clawandGitHub 5c0e2e6185 Merge pull request #12 from life-claw/imgVenueFix
Fixed ImageVenue
2020-07-30 20:56:41 +01:00
death-clawandGitHub 4836d2dacb Merge branch 'master' into imgVenueFix 2020-07-30 20:56:30 +01:00
death-claw bc81ec1ab5 Bug fixes
Code enhancements
2020-07-30 14:59:58 +01:00
death-claw 8a4d039a37 Bug fixes
Code enhancements
2020-07-30 14:00:56 +01:00
death-claw 468639264a Major rewrites of data persistence, websocket, and user interface 2020-07-26 15:10:09 +01:00
death-claw 9b649b470f Major rewrites of data persistence, websocket, and user interface 2020-07-26 14:45:04 +01:00
life-claw edd5cf2bf7 Fixed ImageVenue
Updated XPaths to match their current markup
And added HTTPs redirect, since they support that now
2020-07-17 16:57:51 -05:00
death-claw 67c605e640 v2.12.1 2020-05-22 15:55:05 +01:00
death-claw 8d4b878656 Fix alternative name bug 2020-05-22 15:54:11 +01:00
death-claw bf532dd1d0 v2.12.0 2020-05-21 16:43:03 +01:00
death-claw eb1a125e59 Use new user hash from the api 2020-05-21 16:41:16 +01:00
death-claw 256659a72d v2.11.7 2020-05-08 12:57:41 +01:00
death-claw dbf3aa7e7d Fix bug with paths creation
Move Open Link button
Rename rename all title
2020-05-08 12:56:11 +01:00
death-claw a75ce181ff v2.11.6 2020-05-04 21:07:16 +01:00
death-claw 37ae6b4ebf Merge branch 'master' of https://github.com/death-claw/vripper-project 2020-05-04 21:06:48 +01:00
death-claw 6631a6a95b PNG filenames are now correctly named
Fix issue with special characters in post titles
2020-05-04 21:04:51 +01:00
death-clawandGitHub 918f8578ad Update CHANGELOG.md 2020-05-03 16:40:26 +01:00
death-claw 10fa9f68ae v2.11.5 2020-05-03 15:18:28 +01:00
death-claw 6b5c9ccb0f Reduce rows buffer size 2020-05-03 15:16:31 +01:00
death-claw 652e8f1a9d v2.11.4 2020-05-03 14:24:27 +01:00
death-claw fe53ae733d Code cleanup
Dependencies update
2020-05-03 14:22:40 +01:00
death-claw 690dab2ae8 v2.11.3 2020-04-30 18:26:42 +01:00
death-claw 530542163e Add missing files 2020-04-30 18:26:23 +01:00
death-claw cf7f4c75c0 Source code restructure
UI fixes and enhancements
Add menu entry to rename a gallery to first alternative title
Add menu on toolbar for massive rename to first alternative title
2020-04-30 18:23:11 +01:00
death-claw c6b395cf4c v2.11.2 2020-04-26 22:06:34 +01:00
death-claw e72edeed62 Fix icon
Bug fixes
2020-04-26 22:02:51 +01:00
death-claw 3f22d53f89 v2.11.1 2020-04-25 16:32:25 +01:00
death-claw 08c0fa62b1 Add option to rename gallery
Code source maintenance
2020-04-25 16:28:56 +01:00
death-claw 4c79cca079 v2.11.0 2020-03-29 17:37:28 +01:00
death-clawandGitHub 67096242a2 Merge pull request #4 from life-claw/pixroute
Added Support for pixroute.com
2020-03-29 17:30:34 +01:00
death-claw f66112375d Merge branch 'master' of https://github.com/death-claw/vripper-project 2020-03-29 17:28:39 +01:00
death-claw c5a082e248 Upgrade Electron to v8
Upgrade Angular to v9
App settings are now portable in a json file
Add grab post title from thread option
General code source maintenance
2020-03-29 17:03:14 +01:00
life-claw c2a1b0aa33 Added Support for pixroute.com 2020-03-08 13:25:57 -05:00
death-clawandGitHub a89dcc96d8 Update README.md 2020-02-11 21:16:37 +01:00
death-claw b2a10eafd9 v2.10.8 2020-01-25 16:00:05 +01:00
death-claw f027aff89d Fix app url 2020-01-25 15:57:45 +01:00
death-claw b4eaf0d1c4 v2.10.7 2020-01-25 14:45:54 +01:00
death-claw 9d2975faaa Drop SockJs and switch to WebSocket
Autoreconnect front to back
2020-01-25 14:39:50 +01:00
death-claw 0c0fdf9d78 v2.10.6 2020-01-19 13:01:03 +01:00
death-claw e17afa0ef3 Limit cache size 2020-01-19 12:59:47 +01:00
death-claw 3d3eb6aa36 v2.10.5 2020-01-18 22:28:27 +01:00
death-claw e22132255e Bug fixes 2020-01-18 22:24:53 +01:00
death-claw ef041526ec v2.10.4 2020-01-18 13:27:47 +01:00
death-claw 704311dd9a Zero value for total max download will disable the limit
Add system notification
Bug fixes
2020-01-18 13:21:46 +01:00
death-claw 9617a6f20f v2.10.3 2020-01-16 21:39:59 +01:00
death-claw bc17442961 Some bug fixes
Enhance download queue logic
Clear cache button
Fix the partial status
2020-01-16 21:34:09 +01:00
death-claw 574d6818c1 Refactor PostImgHost 2020-01-14 21:03:35 +01:00
death-clawandGitHub 5c416055ee Merge pull request #3 from life-claw/postimg
Added Support for postimg.cc
2020-01-14 19:01:41 +01:00
death-clawandGitHub fa30458dc6 Merge pull request #1 from life-claw/imagevenue
Add support for imagevenue.com
2020-01-14 18:47:47 +01:00
death-clawandGitHub 638af93225 Merge pull request #2 from life-claw/urlSpaces
Sanitize URLs that contain spaces
2020-01-14 18:14:47 +01:00
life-claw 2c6ca741c7 Added PostImgHost 2020-01-07 16:50:11 -06:00
life-claw d7f2df2540 Sanitize URLs with that contain spaces 2020-01-07 16:18:45 -06:00
life-claw b85ade1dba Added ImageVenueHost 2020-01-07 16:04:04 -06:00
death-claw 1c61a0b34e v2.10.2 2019-12-29 11:35:59 +01:00
death-claw 46b9449969 Upgrade electron to v7
Update the retry policy
2019-12-29 11:33:03 +01:00
death-claw cc1bbd1231 v2.10.1 2019-12-27 23:05:10 +01:00
death-claw b29cba2911 Bug fix 2019-12-27 22:58:33 +01:00
death-claw 13bde3aa2c 2.10.0 2019-12-27 21:27:27 +01:00
death-claw 35b4f5fc97 Add support for imgspice
Fix spring dependency bug
2019-12-27 21:25:33 +01:00
death-claw 1b0482b7d4 v2.9.1 2019-12-27 20:10:31 +01:00
death-claw c9ff6eeab6 Fix build issue 2019-12-27 20:09:16 +01:00
death-claw 5ac57b743d v2.9.0 2019-12-27 20:07:19 +01:00
death-claw f7b219701f Allow many hosts to run at the same time
Add title to gallery
Add global concurrent downloads
Decrease timeout for fast fail
Add counter badge in link collector
2019-12-27 20:03:35 +01:00
death-claw d009aa3f8c v2.8.1 2019-12-24 20:21:29 +01:00
death-claw 078d49f883 v2.8.1 2019-12-24 20:18:22 +01:00
death-claw f1bf517ba5 Fix issue with path creation 2019-12-24 20:15:03 +01:00
death-claw 6269ed6f6e v2.8.0 2019-12-24 19:47:17 +01:00
death-claw 43b8dedbb1 Use pointer cursor on gallery items 2019-12-24 19:42:45 +01:00
death-claw 978be7bd49 Add view photos settings
Some improvements
2019-12-23 22:46:46 +01:00
death-claw 1b7f3aef62 Add missing css and script includes 2019-12-23 19:36:07 +01:00
death-claw 7d24ee4867 Use photoswipe for image gallery 2019-12-23 00:02:13 +01:00
death-claw 269a3475b0 Generate thumbnails on the fly and cache them 2019-12-21 19:17:51 +01:00
death-claw 18e2bc6afc Add Photo Viewer 2019-12-21 17:23:22 +01:00
death-claw c6ef54e115 v2.7.1 2019-12-21 11:24:03 +01:00
death-claw 441b2831fc Add version to title 2019-12-21 11:16:53 +01:00
death-claw 3cf4d60eca v2.7.0 2019-12-21 09:59:17 +01:00
death-claw d228c89a1c Add support for pixxxels
Add missing icon
2019-12-21 09:56:57 +01:00
death-claw f3b592e81b v2.6.0 2019-12-21 08:39:04 +01:00
death-claw a5ff659fb4 Better quality app icon
Implement context menu on download tab
Add stop remove and start button to context menu
Link will not be removed automatically from link collector tab
Some UI fixes
2019-12-21 08:34:56 +01:00
death-claw ab80e2f3b3 v2.5.1 2019-12-09 20:26:52 +01:00
death-claw 2ef082f87e Add hosts column in Link collector grab menu
Disable download button when nothing selected in Link collector grab menu
2019-12-09 19:56:26 +01:00
death-claw a1b97e76a7 v2.5.0 2019-12-08 20:13:58 +01:00
death-claw 185a078b4d Missing change 2019-12-08 20:08:42 +01:00
death-claw 28a4c944ee Add support for multiple links at a time
Add the Link collector feature
Fix bug with previews
2019-12-08 20:07:13 +01:00
death-claw 23d3553d99 Include startup icon
Fix UI layout
2019-12-06 23:09:57 +01:00
death-claw fb09e4c46b v2.4.1 2019-11-26 20:03:10 +01:00
death-claw 4feb6f5fe1 Increase attempts for establishing ws connection between front and back end
Change default download folder to Home folder (installer), App folder (portable)
2019-11-26 20:01:08 +01:00
death-claw df1b9efbbb v2.4.0 2019-11-23 14:50:16 +01:00
death-claw 553924fa1d Pimpandhost Host support 2019-11-23 14:48:05 +01:00
death-claw a90dbd568f v2.3.2 2019-11-23 12:15:42 +01:00
death-claw ca7ca1ce8e Fix a bug with the scan component 2019-11-23 12:14:07 +01:00
death-claw b3ab239a1e v2.3.1 2019-11-22 20:07:41 +01:00
death-claw 6e212c791c Fix connexion reset issue
Fix Post Status when autostart option is selected
Add stop all button
2019-11-22 20:05:14 +01:00
death-claw c6067ef147 v2.3.0 2019-11-21 21:17:25 +01:00
death-claw 8333f0c34e Fix Restoring post status on App startup
Use local sockjs and material icons font
2019-11-21 21:16:24 +01:00
death-claw 75becd8bb4 UI enhancements and overhaul
Stability fixes with HTTP connections
MacOS official support
2019-11-21 20:54:44 +01:00
death-claw 667376301a v2.2.5 2019-11-09 13:26:54 +01:00
death-claw 128e6b68a3 More robust parsing
Fix pixhost and imagezilla to exclude thumbs
2019-11-09 13:16:09 +01:00
death-claw a7c2844ab4 Change Retry duration 2019-11-09 10:13:12 +01:00
death-claw bd0b7d274b Fix macOs paths 2019-10-06 13:33:29 +01:00
death-claw bc57fad52b Remove failing rule 2019-10-06 12:00:01 +01:00
death-claw b81ef53cb9 Add support for macOs 2019-10-06 11:22:48 +01:00
death-claw c182bf0a08 v2.2.4 2019-10-05 18:06:40 +01:00
death-claw 07802b4b13 Add AppImage target 2019-10-05 18:03:32 +01:00
death-claw 50c4d947ed v2.2.3 2019-10-01 23:28:12 +01:00
death-claw b9b20e86dd Strip libjvm.so for debian 2019-10-01 23:27:01 +01:00
death-claw 64b93ce7e1 v2.2.2 2019-10-01 22:59:39 +01:00
death-claw 0bde4399e3 Update product name 2019-10-01 22:57:13 +01:00
death-claw 9b7818c5a7 v2.2.1 2019-10-01 22:44:19 +01:00
death-claw 1f2dcff353 Update dependency list for deb package 2019-10-01 22:42:28 +01:00
death-claw 4d82622fdb v2.2.0 2019-10-01 21:26:27 +01:00
death-claw e7393caf01 Fix 'Start' on pending items
Upgrade to Java 11
Embedd JRE within the app
2019-10-01 21:24:02 +01:00
death-claw 8e5cb958ea v2.1.0 2019-09-30 21:12:56 +01:00
death-claw bc2cf02ec6 Add a create a subfolder per thread option 2019-09-30 21:06:40 +01:00
death-claw 7a8ecb306f v2.0.4 2019-09-27 20:52:25 +01:00
death-claw 76c3901afa Fix some issues with ImxHost 2019-09-27 20:50:51 +01:00
death-claw f16b2227a1 v2.0.3 2019-09-23 17:38:36 +01:00
death-claw d8d7ca5408 Update deb dependencies 2019-09-23 17:36:10 +01:00
death-claw 28c6b71e99 v2.0.2 2019-09-23 16:46:37 +01:00
death-claw c40d3e2377 Add portable mode 2019-09-23 16:43:52 +01:00
death-claw 2217fea0ad v2.0.1 2019-09-22 21:55:30 +01:00
death-claw 94746441b5 Update app name 2019-09-22 21:48:35 +01:00
death-claw c3e861fa8b v2.0.0 2019-09-22 20:58:24 +01:00
death-claw ae1d644dba Update icons 2019-09-22 20:41:32 +01:00
death-claw 1fcb7e4e42 Fix a bug in data file 2019-09-22 20:29:34 +01:00
death-claw 646cbba21c Update package.json 2019-09-22 20:21:05 +01:00
death-claw eef4d13235 Change data and log paths 2019-09-22 20:15:03 +01:00
death-claw 9d1fe4c6b6 Electron app changes 2019-09-22 18:41:27 +01:00
death-claw 610d08f4ca Add sub folder option
Add clear finished option
UI enhancements
Bug fixes
2019-09-22 16:23:52 +01:00
death-claw 7e52d0d6de v1.6.5 2019-09-18 23:20:58 +01:00
death-claw 4f0c5925ad electron update 2019-09-18 23:19:01 +01:00
death-claw 245668b374 v1.6.4 2019-09-18 20:29:18 +01:00
death-claw d3f5a70bfb Change menu item name 2019-09-18 20:27:36 +01:00
death-claw 5fd70a821b v1.6.3 2019-09-18 19:44:45 +01:00
death-claw 6e9c912118 Better multithread management
Fix partial state UI
2019-09-18 19:41:48 +01:00
death-claw 19a98ea056 v1.6.2 2019-09-17 23:00:55 +01:00
death-claw 229bc46227 Rework UI colors
Fix scan input bug
2019-09-17 22:59:04 +01:00
death-claw 374946c6f2 v1.6.1 2019-09-16 22:56:57 +01:00
death-claw f8eccfb4ad Rework UI
Fix clipboard monitoring
2019-09-16 22:54:18 +01:00
death-claw 4223bce5f3 v1.6.0 2019-09-15 22:19:53 +01:00
death-claw ea9a98bb3e Rework the UI
Add a dark theme
Fix the force ordering bug
Other bug fixes
2019-09-15 22:14:17 +01:00
death-claw a416a7e7d7 v1.5.2 2019-09-01 15:46:41 +01:00
death-claw 24dd46ab2c Fix imx host 2019-09-01 15:40:34 +01:00
death-claw 82f4d1bc6a v1.5.1 2019-08-24 19:23:49 +01:00
death-claw 819d844f23 Bug fixes and electron upgrade 2019-08-24 19:19:31 +01:00
death-claw ab4711ffef v1.5.0
- Display logged in user
- Add post details when clicking on main list items (Status, Post URL, Host)
- Add open download location on main list items menu
- Add support for multi-post threads (Show a menu to select posts to download)
- Add link to image from details list
2019-08-24 14:57:43 +01:00
death-claw f107acf376 Merge branch 'master' of https://github.com/death-claw/vripper-project into development 2019-08-16 11:50:24 +01:00
death-claw 43788a0f96 v1.4.10 2019-08-15 18:33:45 +01:00
death-claw 9ee8b90003 Fix image rename on windows 2019-08-15 18:30:58 +01:00
death-claw 54c9a21861 v1.4.9 2019-08-13 15:00:41 +01:00
death-claw fe3543a4ab Update Built-By to be generic 2019-08-13 14:56:19 +01:00
death-claw eeb24ae7c1 v1.4.8 2019-08-13 14:39:41 +01:00
death-claw 9b9c5ac5c8 Add option to force image ordering 2019-08-13 14:34:46 +01:00
death-claw bf2e8c4bb6 v1.4.7 2019-08-13 11:21:54 +01:00
death-claw 5d449fa4e5 Fixed a bug in image filename extensions 2019-08-13 11:07:11 +01:00
death-claw 35cd3c53e1 Implement thread parse with vr api 2019-08-06 20:03:13 +01:00
death-claw 13d1d134f6 INTERMEDIATE COMMIT 2019-08-06 18:24:20 +01:00
death-claw c8b76b2dfb v1.4.6
* Fixed a bug when creating image file names
* Remove unnecessary logs
* Fix data.json init
2019-08-06 16:44:38 +01:00
335 changed files with 24511 additions and 20147 deletions
+3 -1
View File
@@ -2,4 +2,6 @@
/**/*/node_modules
/**/*/target
/**/*/build-dir
.idea
/**/*/java-runtime
.idea
.vripper
+692 -3
View File
@@ -1,65 +1,754 @@
# Changelog
## [1.4.5] - 2019-08-03
## [3.5.4] - 2021-05-29
### Changed
- Add min width and height for electron app
- Fix download queue bug
## [3.5.3] - 2021-05-24
### Changed
- Update donation infos
## [3.5.1] - 2021-05-22
### Changed
- Bug fix
## [3.5.0] - 2021-05-21
### Changed
- Download queue bug fix
- Save window size and position
- Resize confirmation box
- Enhance metadata interruption
- Add Added on and Order columns
## [3.4.2] - 2021-05-10
### Changed
- Fix potential issue with imagebam host
- Enhance download queue logic
- Code source enhancements
- Fix app icons
- Fix race condition bug
## [3.3.8] - 2021-05-05
### Changed
- Fix imagebam host
- Fix font path
## [3.3.6] - 2021-03-20
### Changed
- Optimize build size
## [3.3.5] - 2021-03-18
### Changed
- Bug fixe
## [3.3.4] - 2021-03-18
### Changed
- Bug fixes
## [3.3.3] - 2021-03-13
### Changed
- Remove unnecessary event logs
## [3.3.2] - 2021-03-11
### Changed
- Fix a bug with event logs
- Change event names and add some icons
## [3.3.1] - 2021-02-18
### Added
- Add about menu
### Changed
- Fix README.md
## [3.3.0] - 2021-02-16
### Changed
- Add support for proxies
- Add a Event Logs
- Fix bugs with the database
## [3.2.3] - 2021-01-13
### Changed
- Finished posts are not cleared from the UI
## [3.2.2] - 2021-01-13
### Changed
- Set total download to max value
## [3.2.1] - 2021-01-12
### Added
- Add settings for connection timeout
- Add settings for maximum attempts
### Changed
- Enhance posts rename logic
- Dependencies update
- Switch reactive from rx-java to reactor
## [3.2.0] - 2020-11-16
### Changed
- Fix build issue
## [3.1.0] - 2020-11-16
### Changed
- UI revamp
- Fix bugs
## [3.0.11] - 2020-10-31
### Added
- Support for vipr.im
## [3.0.10] - 2020-10-04
### Changed
- Fix bug on settings update
## [3.0.9] - 2020-09-27
### Changed
- Revert to appending post id to download folders
- Fix a bug that lead to adding duplicate posts
- Remove old unused settings
## [3.0.8] - 2020-08-22
### Changed
- Bug fix which breaks concurrent downloads
## [3.0.7] - 2020-08-22
### Changed
- Fix MacOs build
## [3.0.6] - 2020-08-18
### Changed
- Fix TurboImageHost
## [3.0.5] - 2020-08-17
### Changed
- Update packages
- Host class refactoring
## [3.0.4] - 2020-08-16
### Changed
- Improve rename UI
## [3.0.3] - 2020-08-16
### Changed
- Fix bugs with electron app
## [3.0.2] - 2020-08-15
### Changed
- Fix bugs with download queue
## [3.0.1] - 2020-08-03
### Changed
- Fix bugs with download queue
## [3.0.0] - 2020-08-03
### Changed
- Major rewrites
- Introducing a proper database for persistence
## [2.12.1] - 2020-05-22
### Changed
- Fix alternative name bug
## [2.12.0] - 2020-05-21
### Changed
- Use new user hash from the api
## [2.11.7] - 2020-05-08
### Changed
- Fix bug with paths creation
- Move Open Link button
- Rename rename all title
## [2.11.6] - 2020-05-04
### Changed
- PNG filenames are now correctly named
- Fix issue with special characters in post titles
## [2.11.5] - 2020-05-03
### Changed
- Reduce rows buffer size
## [2.11.4] - 2020-05-03
### Changed
- Code cleanup
- Dependencies update
## [2.11.3] - 2020-04-30
### Changed
- Source code restructure
- UI fixes and enhancements
### Added
- Add menu entry to rename a gallery to first alternative title
- Add menu on toolbar for massive rename to first alternative title
## [2.11.2] - 2020-04-26
### Changed
- Bug fixes
## [2.11.1] - 2020-04-25
### Changed
- Code source maintenance
### Added
- Added option to rename gallery
## [2.11.0] - 2020-03-29
### Changed
- Upgrade Electron to v8
- Upgrade Angular to v9
- App settings are now portable in a json file
- General code source maintenance
### Added
- Added Support for pixroute.com
- Added grab post title from thread option
## [2.10.8] - 2020-01-25
### Changed
- Fix app url
## [2.10.7] - 2020-01-25
### Changed
- Drop SockJs and switch to WebSocket
- Autoreconnect front to back
## [2.10.6] - 2020-01-19
### Changed
- Limit cache size
## [2.10.5] - 2020-01-18
### Changed
- Bug fixes
## [2.10.4] - 2020-01-18
### Changed
- Zero value for total max download will disable the limit
- Add system notification
- Bug fixes
## [2.10.3] - 2020-01-16
### Changed
- Some bug fixes
- Enhance download queue logic
- Clear cache button
- Fix the partial status
- Sanitize URLs with spaces
### Added
- Support for Postimg host
- Support for Imagevenue host
## [2.10.2] - 2019-12-29
### Changed
- Upgrade electron to v7
- Bug fix
## [2.10.1] - 2019-12-27
### Changed
- Bug fix
## [2.10.0] - 2019-12-27
### Changed
- Fix spring dependency bug
### Added
- Add support for imgspice
## [2.9.0] - 2019-12-27
### Changed
- Fix build issue
## [2.9.0] - 2019-12-27
### Changed
- Decrease timeout for fast fail
### Added
- Allow many hosts to run at the same time
- Add title to gallery
- Add global concurrent downloads
- Add counter badge in link collector
## [2.8.1] - 2019-12-24
### Changed
- Fix issue with path creation
## [2.8.0] - 2019-12-24
### Added
- Photo gallery feature
## [2.7.1] - 2019-12-21
### Added
- Add version to title bar
## [2.7.0] - 2019-12-21
### Changed
- Add missing icon
### Added
- Add support for pixxxels
## [2.6.0] - 2019-12-21
### Changed
- Disable download button when nothing selected in Link collector grab menu
- Better quality app icon
- Implement context menu on download tab
- Add stop remove and start button to context menu
- Link will not be removed automatically from link collector tab
- Some UI fixes
## [2.5.1] - 2019-12-09
### Changed
- Disable download button when nothing selected in Link collector grab menu
### Added
- Add hosts column in Link collector grab menu
## [2.5.0] - 2019-12-08
### Changed
- Change loading screen
- UI Fixes
- Fix bug with previews
### Added
- Add link collector feature
- Add support for multiple links at a time
## [2.4.1] - 2019-11-26
### Changed
- Increase attempts for establishing ws connection between front and back end
- Change default download folder to Home folder (installer), App folder (portable)
## [2.4.0] - 2019-11-23
### Changed
- Pimpandhost Host support
## [2.3.2] - 2019-11-23
### Changed
- Fix a bug with the scan component
## [2.3.1] - 2019-11-22
### Changed
- Fix connexion reset issue
- Fix Post Status when autostart option is selected
- Add stop all button
## [2.3.0] - 2019-11-21
### Changed
- UI enhancements and overhaul
- Stability fixes with HTTP connections
- MacOS official support
- Use local sockjs and material icons font
## [2.2.5] - 2019-11-09
### Changed
- Exclude thumbs download for imagezilla and pixhost
- Change retry on fail logic (may help avoid having download errors)
## [2.2.4] - 2019-10-05
### Added
- AppImage target for linux
## [2.2.3] - 2019-10-01
### Changed
- Strip libjvm.so for debian
## [2.2.2] - 2019-10-01
### Changed
- Update product name
## [2.2.1] - 2019-10-01
### Changed
- Update dependency list for deb package
## [2.2.0] - 2019-10-01
### Added
- Embedd JRE within the app
### Changed
- Fix 'Start' on pending items
- Upgrade to Java 11
## [2.1.0] - 2019-09-30
### Added
- Add a create a subfolder per thread option
## [2.0.4] - 2019-09-27
### Changed
- Fix some issues with ImxHost
## [2.0.3] - 2019-09-23
### Changed
- Update deb dependencies
## [2.0.2] - 2019-09-23
### Changed
- Add portable mode
## [2.0.1] - 2019-09-22
### Changed
- Change app name
## [2.0.0] - 2019-09-22
### Added
- Add sub folder option
- Add clear finished option
### Changed
- UI enhancements
- Bug fixes
- Change data and log paths
## [1.6.5] - 2019-09-18
### Changed
- electron update
## [1.6.4] - 2019-09-18
### Changed
- Change menu item name
## [1.6.3] - 2019-09-18
### Changed
- Better multithread management
- Fix partial state UI
## [1.6.2] - 2019-09-17
### Changed
- Rework UI colors
- Fix scan input bug
## [1.6.1] - 2019-09-16
### Changed
- Rework UI
- Fix clipboard monitoring
## [1.6.0] - 2019-09-15
### Changed
- Rework the UI
- Add a dark theme
- Fix the force ordering bug
- Other bug fixes
## [1.5.2] - 2019-09-01
### Added
- imx.to host fix
## [1.5.1] - 2019-08-24
### Added
- Bug fixes
## [1.5.0] - 2019-08-24
### Added
- Display logged in user
- Add post details when clicking on main list items (Status, Post URL, Host)
- Add open download location on main list items menu
- Add support for multi-post threads (Show a menu to select posts to download)
- Add link to image from details list
## [1.4.10] - 2019-08-15
### Changed
- Fix a rename issue on windows
## [1.4.9] - 2019-08-13
### Changed
- Update Built-By to be generic
## [1.4.8] - 2019-08-13
### Changed
- Add option to force image ordering
## [1.4.7] - 2019-08-13
### Changed
- Fixed a bug in image filename extensions
## [1.4.6] - 2019-08-06
### Changed
- Fixed a bug when creating image file names
- Remove unnecessary logs
- Fix data.json init
## [1.4.5] - 2019-08-03
### Changed
- Add min width and height for electron app
### Added
- Add download speed to status bar
## [1.4.4] - 2019-08-01
### Changed
- Fix names for ImgboxHost
- Enhance UI
- Minor fixes
### Added
- Add start and stop button
## [1.4.3] - 2019-07-14
### Changed
- Update some messages
- Fix scrolling issue
## [1.4.2] - 2019-07-14
### Changed
- Fix layout issue in web mode
## [1.4.1] - 2019-07-14
### Changed
- Change title's font
### Added
- Add borders
## [1.4.0] - 2019-07-14
### Changed
- Better error handling on startup
- Better support for linux AppImage
### Added
- Clear all completed
- Add clipboard support
- Add option to remove all posts
- Add confirmation on remove
## [1.3.0-rc0] - 2019-07-04
### Changed
- Change default setting for autostart false -> true
- Change default setting for max concurrent downloads 1 -> 4
### Added
- Desktop app built with electron
- Add the option to remove a post
## [1.2.0] - 2019-06-23
### Added
- Add a shutdown button to gracefully shutdown the app
- Open and redirect the browser to the app address on startup
- Add the option to run the app in headless mode (-Djava.awt.headless=true)
## [1.1.1] - 2019-06-22
### Changed
- Fix a stability issue with progress updates when the app is runninh for a long time
## [1.0.0] - 2019-06-22
### Added
- Add real time communication for progress updates via websocket (SockJS)
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
+21 -12
View File
@@ -1,26 +1,35 @@
# Viper Ripper!
# VRipper!
This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
This is my spin for a cross platform gallery ripper app for [vipergirls](https://vipergirls.to) website. The purpose of
this project is to build a robust and clean application to conveniently download photo galleries using modern web
technologies, Java + Spring boot for the back end and angular + electron for the front end.
## How to build
You need a recent version of maven 3.6.1+.
The application support 2 build modes:
- Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
- Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
- Desktop app: Self contained app, built with electron includes Java runtimes.
- Server app: Depends on Java runtimes, you need a web browser to access the app UI.
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there
for you.
To build the server app:
mvn clean install
mvn clean install -DskipTests
To build the desktop app:
mvn clean install -Pelectron
mvn clean install -Pelectron -DskipTests
Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
Maven will automatically handle front end compilation. However, for development, you will need to install a recent
version of nodejs on your system.
*Partially cross platform:
Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.
If you feel generous and want to support me, you can donate some coins, and I would be very grateful 🙏
I accept [PayPal](https://www.paypal.com/myaccount/transfer/homepage/buy/preview) donations, following is my email
address
dev.vripper@gmail.com
+6 -6
View File
@@ -2,19 +2,19 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>1.4.6-SNAPSHOT</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.1.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
<version>3.5.6</version>
<relativePath/>
</parent>
<modules>
<module>vripper-server</module>
<module>vripper-ui</module>
<module>vripper-electron</module>
</modules>
</project>
</project>
Binary file not shown.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 303 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 592 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

+153 -109
View File
@@ -1,13 +1,17 @@
const { app, BrowserWindow } = require("electron");
require('v8-compile-cache');
const {app, BrowserWindow} = require("electron");
const path = require("path");
const url = require("url");
const getPort = require("get-port");
const { spawn } = require("child_process");
const { ipcMain } = require("electron");
const commandExists = require("command-exists").sync;
const { dialog } = require("electron");
const {spawn} = require("child_process");
const {ipcMain} = require("electron");
const {dialog} = require("electron");
const axios = require('axios');
const appDir = process.env.APPDIR;
const windowStateKeeper = require('electron-window-state');
// non null value when it is an AppImage
const appImageDir = process.env.APPDIR;
const appImagePath = process.env.APPIMAGE;
let win;
let vripperServer;
@@ -19,120 +23,160 @@ let terminationInteval;
let terminated = false;
process.on("uncaughtException", err => {
dialog.showErrorBox(err.message, err.stack);
process.exit(1);
dialog.showErrorBox(err.message, err.stack);
process.exit(1);
});
if (!commandExists("java")) {
dialog.showErrorBox(
"Java command is missing",
"Java cannot be found on your PATH, make sure to install java before running Viper Ripper"
);
process.exit(1);
createWindow = () => {
if (process.platform === 'win32') {
app.setAppUserModelId('me.vripper.vripper-app');
}
let icon;
if (process.platform === 'win32') {
icon = __dirname + '/icon.ico';
} else if (process.platform === 'linux') {
icon = __dirname + '/icon.png';
}
const mainWindowState = windowStateKeeper({
defaultWidth: 1024,
defaultHeight: 800
});
win = new BrowserWindow({
x: mainWindowState.x,
y: mainWindowState.y,
width: mainWindowState.width,
height: mainWindowState.height,
minWidth: 800,
minHeight: 600,
webPreferences: {
nodeIntegration: true,
enableRemoteModule: true
},
icon: icon
});
mainWindowState.manage(win);
win.removeMenu();
win.setMenu(null);
win.setMenuBarVisibility(false);
win.loadURL(
url.format({
pathname: path.join(__dirname, `vripper-ui/index.html`),
protocol: "file:",
slashes: true
})
);
// win.webContents.openDevTools();
win.on("closed", () => {
win = null;
});
}
function createWindow() {
let icon;
if(process.platform === "win32") {
icon = path.join(__dirname, `icon.ico`);
} else {
icon = path.join(__dirname, `icon.icns`);
}
win = new BrowserWindow({
width: 1024,
height: 768,
minWidth: 640,
minHeight: 480,
frame: false,
webPreferences: {
nodeIntegration: true
},
icon: icon
});
win.removeMenu();
win.loadURL(
url.format({
pathname: path.join(__dirname, `vripper-ui/index.html`),
protocol: "file:",
slashes: true
})
);
// win.webContents.openDevTools();
win.on("closed", () => {
win = null;
});
shutdownServer = () => {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: {'content-type': 'application/json'},
}).then((response) => {
terminationInteval = setInterval(() => {
terminationAttemps++;
if (terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
app.quit();
} else if (terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
}).catch((error) => {
// Terminate immediately
console.log(error);
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
getPort().then(port => {
serverPort = port;
ipcMain.on("get-port", event => {
event.reply("port", port);
});
vripperServer = spawn("java", [
"-Dvripper.server.port=" + port,
"-jar",
appDir !== undefined ? path.join(appDir, "bin/vripper-server.jar") :
path.join(app.getPath('exe'), "../bin/vripper-server.jar")
], {
stdio: 'ignore'
});
vripperServer.on('exit', (code, signal) => {
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
terminated = true;
app.quit();
});
});
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
app.quit();
app.quit();
} else {
app.on("second-instance", (event, commandLine, workingDirectory) => {
if (win) {
if (win.isMinimized()) win.restore();
win.focus();
}
});
getPort().then(port => {
serverPort = port;
ipcMain.on("get-port", event => {
event.reply("port", port);
});
app.on("ready", createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
headers: { 'content-type': 'application/json' },
}).then((response) => {
terminationInteval= setInterval(() => {
terminationAttemps++;
if(terminated) {
console.log('viper server terminated');
clearInterval(terminationInteval);
const appPath = path.join(app.getAppPath(), '../../');
let javaBinPath, jarPath, baseDir;
if (appImageDir !== undefined && process.platform === 'linux') {
javaBinPath = path.join(appImageDir, "java-runtime/bin/java");
jarPath = path.join(appImageDir, "bin/vripper-server.jar");
baseDir = path.join(appImagePath, '..');
} else if (process.platform === 'darwin') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = app.getPath('appData');
} else if (process.platform === 'win32') {
javaBinPath = path.join(appPath, "java-runtime/bin/java");
jarPath = path.join(appPath, "bin/vripper-server.jar");
baseDir = appPath;
} else {
console.error(`Unknown platform ${process.platform}`);
app.quit();
} else if(terminationAttemps > maxTerminationAttemps) {
console.log('viper server is not terminated');
console.log('Proceed to kill');
vripperServer.kill('SIGKILL');
clearInterval(terminationInteval);
app.quit();
}
}, 1000);
})
.catch((error) => {
// Terminate immediately
vripperServer.kill('SIGKILL');
terminated = true;
app.quit();
});
}
});
}
app.on("activate", () => {
if (win === null) {
createWindow();
}
});
vripperServer = spawn(javaBinPath, [
"-Xms256m",
"-Dvripper.server.port=" + port,
"-Dbase.dir=" + baseDir,
"-Djava.net.preferIPv4Stack=true",
"-jar",
jarPath
], {
stdio: 'ignore'
});
vripperServer.on('exit', (code, signal) => {
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
terminated = true;
app.quit();
});
});
app.on("second-instance", (event, commandLine, workingDirectory) => {
if (win) {
if (win.isMinimized()) win.restore();
win.focus();
}
});
app.on("ready", createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
shutdownServer();
}
});
app.on("will-quit", () => {
if (process.platform === "darwin") {
shutdownServer();
}
});
app.on("activate", () => {
if (win === null) {
createWindow();
}
});
}
-3269
View File
File diff suppressed because it is too large Load Diff
+40 -17
View File
@@ -1,12 +1,14 @@
{
"name": "vripper-electron",
"version": "1.4.6-SNAPSHOT",
"description": "",
"version": "3.6.0",
"description": "A ripper for vipergirls.to built using web technolgies",
"main": "main.js",
"author": "",
"author": "dev-claw",
"homepage": "https://github.com/dev-claw/vripper-project",
"license": "ISC",
"build": {
"appId": "tn.mnlr.vripper",
"appId": "me.vripper.vripper-app",
"productName": "VRipper",
"files": [
"**/*",
"!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme}",
@@ -20,6 +22,10 @@
"!**/{__pycache__,thumbs.db,.flowconfig,.idea,.vs,.nyc_output}",
"!**/{appveyor.yml,.travis.yml,circle.yml}",
"!**/{npm-debug.log,yarn.lock,.yarn-integrity,.yarn-metadata.json}",
"!**/build-dir",
"!**/java-runtime",
"!**/pre-build.js",
"!**/pom.xml",
{
"from": "./build/",
"to": "."
@@ -29,33 +35,50 @@
{
"from": "../vripper-server/target/vripper-server-${version}-electron.jar",
"to": "bin/vripper-server.jar"
},
{
"from": "java-runtime",
"to": "java-runtime"
}
],
"win": {
"target": "nsis"
"icon": "icon.ico",
"target": [
"dir"
]
},
"linux": {
"synopsis": "vipergirls.to ripper",
"category": "Utility",
"packageCategory": "Utility",
"target": [
"zip"
"AppImage"
]
},
"mac": {
"category": "public.app-category.utilities",
"target": [
"dmg"
],
"icon": "icon.icns"
}
},
"scripts": {
"start": "electron .",
"dist": "node pre-build.js && build"
"dist": "node pre-build.js && electron-builder"
},
"devDependencies": {
"electron": "^5.0.5",
"electron-builder": "^20.44.4"
"electron": "11.1.1",
"electron-builder": "22.9.1"
},
"dependencies": {
"axios": "^0.19.0",
"cheerio": "^1.0.0-rc.3",
"command-exists": "^1.2.8",
"copy-dir": "^1.1.0",
"custom-electron-titlebar": "^3.0.10",
"electron-context-menu": "^0.13.0",
"get-port": "^5.0.0",
"rimraf": "^2.6.3"
"axios": "0.21.1",
"cheerio": "1.0.0-rc.3",
"copy-dir": "1.3.0",
"electron-context-menu": "2.1.0",
"electron-window-state": "^5.0.3",
"get-port": "5.1.1",
"rimraf": "3.0.2",
"v8-compile-cache": "2.1.1"
}
}
+72 -71
View File
@@ -1,77 +1,78 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>1.4.6-SNAPSHOT</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<classifier>${buildClassifier}</classifier>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>tn.mnlr</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
</parent>
<artifactId>vripper-electron</artifactId>
<name>vripper-electron</name>
<packaging>pom</packaging>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.3</version>
<configuration>
<nodeVersion>v12.2.0</nodeVersion>
<npmVersion>6.9.0</npmVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>npm run build</id>
<goals>
<goal>npm</goal>
</goals>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<arguments>run dist</arguments>
<classifier>${buildClassifier}</classifier>
</configuration>
<phase>generate-resources</phase>
</execution>
</executions>
</plugin>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</build>
<profiles>
<profile>
<id>electron</id>
<properties>
<buildClassifier>electron</buildClassifier>
</properties>
<dependencies>
<dependency>
<groupId>me.vripper</groupId>
<artifactId>vripper-server</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.11.2</version>
<configuration>
<nodeVersion>v14.16.0</nodeVersion>
<yarnVersion>v1.22.10</yarnVersion>
<workingDirectory>build-dir</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and yarn</id>
<goals>
<goal>install-node-and-yarn</goal>
</goals>
</execution>
<execution>
<id>yarn install</id>
<goals>
<goal>yarn</goal>
</goals>
</execution>
<execution>
<id>yarn run build</id>
<goals>
<goal>yarn</goal>
</goals>
<configuration>
<arguments>run dist</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
+8 -1
View File
@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
const fs = require('fs');
const rimraf = require("rimraf");
const copydir = require("copy-dir");
const {execSync} = require("child_process");
rimraf.sync('./build/vripper-ui');
copydir.sync('../vripper-ui/dist/vripper-ui', './build/vripper-ui', {});
@@ -16,3 +16,10 @@ $('body').prepend(`<script>require('../renderer.js')</script>`);
fs.writeFileSync('./build/vripper-ui/index.html', $.html());
console.log('Building runtime environment');
rimraf.sync('java-runtime');
execSync('jlink --no-header-files --no-man-pages --compress=2 --strip-debug --add-modules java.base,java.desktop,java.instrument,java.management,java.security.jgss,java.sql,jdk.unsupported,jdk.crypto.ec --output java-runtime');
if (process.platform === 'linux') {
console.log('Stripping libjvm.so');
execSync('strip -p --strip-unneeded java-runtime/lib/server/libjvm.so');
}
+1 -10
View File
@@ -1,12 +1,3 @@
console.log("renderer intialized");
const customTitlebar = require("custom-electron-titlebar");
const electron = require("electron").remote;
const contextMenu = require("electron-context-menu");
contextMenu({});
const menu = new electron.Menu();
const titlebar = new customTitlebar.Titlebar({
backgroundColor: customTitlebar.Color.fromHex("#3f51b5"),
menu: menu
});
titlebar.updateTitle("Viper Ripper");
contextMenu({});
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
File diff suppressed because it is too large Load Diff
+50 -63
View File
@@ -1,29 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tn.mnlr</groupId>
<artifactId>vripper</artifactId>
<version>1.4.6-SNAPSHOT</version>
<groupId>me.vripper</groupId>
<artifactId>vripper-parent</artifactId>
<version>3.6.0</version>
</parent>
<artifactId>vripper-server</artifactId>
<name>vripper-server</name>
<description>vripper-server</description>
<properties>
<java.version>1.8</java.version>
<java.version>11</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jersey</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
@@ -32,6 +24,19 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -50,17 +55,16 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
<version>4.5.14</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.22</version>
<version>2.24</version>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
<artifactId>rxjava</artifactId>
<version>2.2.6</version>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
</dependency>
<dependency>
<groupId>net.jodah</groupId>
@@ -68,20 +72,45 @@
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>tn.mnlr</groupId>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>me.vripper</groupId>
<artifactId>vripper-ui</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<type>pom</type>
<classifier>${buildClassifier}</classifier>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.14.0</version>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.4.2</version>
<configuration>
<classifier>${buildClassifier}</classifier>
<archive>
<manifestEntries>
<Build-Time>${maven.build.timestamp}</Build-Time>
<Built-By>${user.name}</Built-By>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
@@ -99,52 +128,10 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
</execution>
<execution>
<id>validate-the-git-infos</id>
<goals>
<goal>validateRevision</goal>
</goals>
</execution>
</executions>
<configuration>
<dotGitDirectory>${project.parent.basedir}/.git</dotGitDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<archive>
<manifestEntries>
<Build-Time>${maven.build.timestamp}</Build-Time>
</manifestEntries>
<manifestSections>
<manifestSection>
<name>Versions</name>
<manifestEntries>
<Implementation-Version>${git.build.version}</Implementation-Version>
<Implementation-SCM-Revision>${git.commit.id}</Implementation-SCM-Revision>
<Implementation-SCM-Branch>${git.branch}</Implementation-SCM-Branch>
<Implementation-SCM-Time>${git.build.time}</Implementation-SCM-Time>
<Implementation-SCM-Dirty>${git.dirty}</Implementation-SCM-Dirty>
<Implementation-SCM-Tags>${git.tags}</Implementation-SCM-Tags>
</manifestEntries>
</manifestSection>
</manifestSections>
</archive>
</configuration>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
</plugins>
</build>
@@ -0,0 +1,31 @@
package me.vripper;
import lombok.Getter;
import me.vripper.jpa.repositories.IPostRepository;
import me.vripper.services.DataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
@Component
public class EventListenerBean {
@Getter private static boolean init = false;
private final IPostRepository postRepository;
private final DataService dataService;
@Autowired
public EventListenerBean(IPostRepository postRepository, DataService dataService) {
this.postRepository = postRepository;
this.dataService = dataService;
}
@EventListener
public void onApplicationEvent(ContextRefreshedEvent event) {
postRepository.setDownloadingToStopped();
dataService.sortPostsByRank();
init = true;
}
}
@@ -0,0 +1,38 @@
package me.vripper;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class SpringContext implements ApplicationContextAware {
private static ConfigurableApplicationContext context;
public static <T> T getBean(Class<T> beanClass) {
return context.getBean(beanClass);
}
public static <T> Map<String, T> getBeansOfType(Class<T> beanClass) {
return context.getBeansOfType(beanClass);
}
public static void close() {
log.info("Application terminating...");
if (context != null) {
context.close();
}
}
@Override
public void setApplicationContext(ApplicationContext context) throws BeansException {
// store ApplicationContext reference to access required beans later on
SpringContext.context = ((ConfigurableApplicationContext) context);
}
}
@@ -0,0 +1,23 @@
package me.vripper;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Utils {
public static String throwableToString(Throwable th) {
try (StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter)) {
th.printStackTrace(printWriter);
printWriter.flush();
stringWriter.flush();
return stringWriter.toString();
} catch (IOException e) {
log.error("Failed to convert throwable to string", e);
}
return "Cannot display error, please check the log file for details";
}
}
@@ -0,0 +1,19 @@
package me.vripper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@Slf4j
public class VripperApplication {
public static void main(String[] args) {
try {
Runtime.getRuntime().addShutdownHook(new Thread(SpringContext::close));
SpringApplication.run(VripperApplication.class, args);
} catch (Exception e) {
log.error("Failed to run the application", e);
}
}
}
@@ -0,0 +1,294 @@
package me.vripper.download;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.services.*;
import me.vripper.services.domain.Settings;
import net.jodah.failsafe.function.CheckedRunnable;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.util.EntityUtils;
@Slf4j
public class DownloadJob implements CheckedRunnable {
private static final Object LOCK = new Object();
private static final int READ_BUFFER_SIZE = 8192;
private final DataService dataService;
private final PathService pathService;
private final ConnectionService cm;
private final VGAuthService authService;
private final DownloadSpeedService downloadSpeedService;
private final Settings settings;
private final HttpClientContext context;
@Getter private final Image image;
@Getter private final Post post;
private volatile boolean stopped = false;
@Getter private volatile boolean finished = false;
DownloadJob(Post post, Image image, Settings settings) {
this.image = image;
this.post = post;
this.settings = settings;
dataService = SpringContext.getBean(DataService.class);
pathService = SpringContext.getBean(PathService.class);
cm = SpringContext.getBean(ConnectionService.class);
authService = SpringContext.getBean(VGAuthService.class);
downloadSpeedService = SpringContext.getBean(DownloadSpeedService.class);
context = HttpClientContext.create();
context.setCookieStore(new BasicCookieStore());
context.setAttribute(
ContextAttributes.OPEN_CONNECTION.toString(),
Collections.synchronizedList(new ArrayList<AbstractExecutionAwareRequest>()));
}
public void download(final Post post, final Image image) throws DownloadException {
try {
image.setStatus(Status.DOWNLOADING);
image.setCurrent(0);
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
synchronized (LOCK) {
if (!post.getStatus().equals(Status.DOWNLOADING)
&& !post.getStatus().equals(Status.PARTIAL)) {
post.setStatus(Status.DOWNLOADING);
dataService.updatePostStatus(post.getStatus(), post.getId());
}
// The post may be updated and the download directory might be set by another thread
Post updatedPost = dataService.findById(post.getId()).orElseThrow();
if (updatedPost.getDownloadDirectory() == null) {
pathService.createDefaultPostFolder(updatedPost, settings);
}
if (settings.getLeaveThanksOnStart() != null && settings.getLeaveThanksOnStart()) {
authService.leaveThanks(updatedPost);
}
}
if (stopped) {
return;
}
/*
* HOST SPECIFIC
*/
log.debug(
String.format(
"Getting image url and name from %s using %s", image.getUrl(), image.getHost()));
HostService.NameUrl nameAndUrl = image.getHost().getNameAndUrl(image, context);
log.debug(String.format("Resolved name for %s: %s", image.getUrl(), nameAndUrl.getName()));
log.debug(
String.format("Resolved image url for %s: %s", image.getUrl(), nameAndUrl.getUrl()));
/*
* END HOST SPECIFIC
*/
if (stopped) {
return;
}
String formatImageFileName = pathService.formatImageFileName(nameAndUrl.getName());
log.debug(
String.format(
"Sanitizing image name from %s to %s", nameAndUrl.getName(), formatImageFileName));
nameAndUrl = new HostService.NameUrl(formatImageFileName, nameAndUrl.getUrl());
HttpClient client = cm.getClient().build();
log.debug(String.format("Downloading %s", nameAndUrl.getUrl()));
HttpGet httpGet = cm.buildHttpGet(nameAndUrl.getUrl(), context);
httpGet.addHeader("Referer", image.getUrl());
try (CloseableHttpResponse response =
(CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
EntityUtils.consumeQuietly(response.getEntity());
throw new DownloadException(
String.format("Server returned code %d", response.getStatusLine().getStatusCode()));
}
if (stopped) {
return;
}
File outputFile = Files.createTempFile("vripper", "tmp").toFile();
try (InputStream downloadStream = response.getEntity().getContent();
FileOutputStream fos = new FileOutputStream(outputFile)) {
if (stopped) {
return;
}
image.setTotal(response.getEntity().getContentLength());
dataService.updateImageTotal(image.getTotal(), image.getId());
log.debug(String.format("%s length is %d", nameAndUrl.getUrl(), image.getTotal()));
log.debug(String.format("Starting data transfer for %s", nameAndUrl.getUrl()));
byte[] buffer = new byte[READ_BUFFER_SIZE];
int read;
while ((read = downloadStream.read(buffer, 0, READ_BUFFER_SIZE)) != -1 && !stopped) {
fos.write(buffer, 0, read);
image.increase(read);
downloadSpeedService.increase(read);
dataService.updateImageCurrent(image.getCurrent(), image.getId());
}
fos.flush();
EntityUtils.consumeQuietly(response.getEntity());
if (stopped) {
return;
}
}
checkImageTypeAndRename(
dataService.findById(post.getId()).orElseThrow(),
outputFile,
nameAndUrl.getName(),
image.getIndex());
}
} catch (Exception e) {
if (stopped) {
return;
}
throw new DownloadException(e);
} finally {
if (image.getCurrent() == image.getTotal() && image.getTotal() > 0) {
image.setStatus(Status.COMPLETE);
} else if (stopped) {
image.setStatus(Status.STOPPED);
} else {
image.setStatus(Status.ERROR);
}
dataService.updateImageStatus(image.getStatus(), image.getId());
finished = true;
}
}
private void checkImageTypeAndRename(Post post, File outputFile, String imageName, int index)
throws HostException {
try (ImageInputStream iis = ImageIO.createImageInputStream(outputFile)) {
Iterator<ImageReader> it = ImageIO.getImageReaders(iis);
if (!it.hasNext()) {
throw new HostException("Image file is not recognized!");
}
ImageReader reader = it.next();
if (reader.getFormatName().equalsIgnoreCase("JPEG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_jpg") && !imageNameLC.endsWith("_jpeg")) {
imageName += ".jpg";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_jpg")) {
toReplace = "_jpg";
} else if (imageNameLC.endsWith("_jpeg")) {
toReplace = "_jpeg";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".jpg";
}
}
} else if (reader.getFormatName().equalsIgnoreCase("PNG")) {
String imageNameLC = imageName.toLowerCase();
if (!imageNameLC.endsWith("_png")) {
imageName += ".png";
} else {
String toReplace = null;
if (imageNameLC.endsWith("_png")) {
toReplace = "_png";
}
if (toReplace != null) {
imageName = imageName.substring(0, imageName.length() - toReplace.length()) + ".png";
}
}
}
} catch (Exception e) {
throw new HostException("Failed to guess image format", e);
}
try {
pathService.getDirectoryAccess().lock();
File downloadDestinationFolder = pathService.calcDownloadDirectory(post, settings);
File outImage =
new File(
downloadDestinationFolder,
(settings.getForceOrder() ? String.format("%03d_", index) : "") + imageName);
Files.copy(outputFile.toPath(), outImage.toPath(), StandardCopyOption.REPLACE_EXISTING);
} catch (Exception e) {
throw new HostException("Failed to rename the image", e);
} finally {
pathService.getDirectoryAccess().unlock();
try {
Files.delete(outputFile.toPath());
} catch (IOException e) {
log.warn(String.format("Failed to delete temporary file %s", outputFile.getAbsolutePath()));
}
}
}
@Override
public void run() throws Exception {
if (stopped) {
finished = true;
return;
}
log.debug(String.format("Starting downloading %s", image.getUrl()));
download(post, image);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadJob that = (DownloadJob) o;
return Objects.equals(image, that.image) && Objects.equals(post, that.post);
}
@Override
public int hashCode() {
return Objects.hash(image, post);
}
public void stop() {
this.stopped = true;
List<AbstractExecutionAwareRequest> requests =
(List<AbstractExecutionAwareRequest>)
this.context.getAttribute(ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
for (AbstractExecutionAwareRequest request : requests) {
request.abort();
}
}
}
public enum ContextAttributes {
OPEN_CONNECTION("OPEN_CONNECTION");
private final String value;
ContextAttributes(String value) {
this.value = value;
}
@Override
public String toString() {
return value;
}
}
}
@@ -0,0 +1,69 @@
package me.vripper.download;
import java.time.LocalDateTime;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.Utils;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.ConnectionService;
import me.vripper.services.DataService;
import net.jodah.failsafe.Failsafe;
@Slf4j
public class DownloadJobWrapper implements Runnable {
private final DownloadService downloadService;
private final DataService dataService;
private final ConnectionService connectionService;
private final ILogEventRepository eventRepository;
private final DownloadJob downloadJob;
public DownloadJobWrapper(final DownloadJob downloadJob) {
downloadService = SpringContext.getBean(DownloadService.class);
dataService = SpringContext.getBean(DataService.class);
connectionService = SpringContext.getBean(ConnectionService.class);
eventRepository = SpringContext.getBean(ILogEventRepository.class);
this.downloadJob = downloadJob;
}
@Override
public void run() {
Failsafe.with(connectionService.getRetryPolicy())
.onFailure(
e -> {
try {
LogEvent logEvent =
new LogEvent(
LogEvent.Type.DOWNLOAD,
LogEvent.Status.ERROR,
LocalDateTime.now(),
String.format(
"Failed to download %s\n %s",
downloadJob.getImage().getUrl(),
Utils.throwableToString(e.getFailure())));
eventRepository.save(logEvent);
} catch (Exception exp) {
log.error("Failed to save event", exp);
}
log.error(
String.format(
"Failed to download %s after %d tries",
downloadJob.getImage().getUrl(), e.getAttemptCount()),
e.getFailure());
downloadJob.getImage().setStatus(Status.ERROR);
dataService.updateImageStatus(
downloadJob.getImage().getStatus(), downloadJob.getImage().getId());
})
.onComplete(
e -> {
dataService.afterJobFinish(downloadJob.getImage(), downloadJob.getPost());
downloadService.afterJobFinish(downloadJob);
log.debug(String.format("Finished downloading %s", downloadJob.getImage().getUrl()));
})
.run(downloadJob);
}
}
@@ -0,0 +1,273 @@
package me.vripper.download;
import jakarta.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.services.DataService;
import me.vripper.services.MetadataService;
import me.vripper.services.SettingsService;
import me.vripper.services.domain.Settings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class DownloadService {
private final int MAX_POOL_SIZE = 12;
private final SettingsService settingsService;
private final DataService dataService;
private final MetadataService metadataService;
private final List<Host> hosts;
private final Map<Host, AtomicInteger> threadCount = new HashMap<>();
private final ExecutorService executor = Executors.newFixedThreadPool(MAX_POOL_SIZE);
private final List<DownloadJob> running = new ArrayList<>();
private final List<DownloadJob> pending = new ArrayList<>();
private final Thread pollThread;
@Autowired
public DownloadService(
SettingsService settingsService,
DataService dataService,
MetadataService metadataService,
List<Host> hosts) {
this.settingsService = settingsService;
this.dataService = dataService;
this.metadataService = metadataService;
this.hosts = hosts;
pollThread =
new Thread(
() -> {
while (!Thread.interrupted()) {
try {
synchronized (this) {
List<DownloadJob> accepted = new ArrayList<>();
List<DownloadJob> candidates = getCandidates(candidateCount());
candidates.forEach(
c -> {
if (canRun(c.getImage().getHost())) {
accepted.add(c);
}
});
pending.removeAll(accepted);
accepted.forEach(this::schedule);
accepted.clear();
this.wait();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
},
"Download scheduler thread");
}
@PostConstruct
private void init() {
pollThread.start();
}
public void destroy() throws Exception {
log.info("Shutting down ExecutionService");
pollThread.interrupt();
executor.shutdown();
dataService
.findAllPosts()
.forEach(
p -> {
log.debug(String.format("Stopping download jobs for %s", p));
this.stopRunning(p.getPostId());
});
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
log.warn("Some jobs are still running!, forcing shutdown");
executor.shutdownNow();
}
}
private synchronized void stopRunning(@NonNull String postId) {
List<DownloadJob> stopping = new ArrayList<>();
Iterator<DownloadJob> iterator = running.iterator();
while (iterator.hasNext()) {
DownloadJob downloadJob = iterator.next();
if (postId.equals(downloadJob.getPost().getPostId())) {
downloadJob.stop();
iterator.remove();
stopping.add(downloadJob);
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(DownloadJob::isFinished);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
public void stopAll(List<String> postIds) {
stop(
Objects.requireNonNullElseGet(
postIds,
() ->
dataService.findAllPosts().stream()
.map(Post::getPostId)
.collect(Collectors.toList())));
}
public void restartAll(List<String> posIds) {
restart(
Objects.requireNonNullElseGet(
posIds,
() ->
dataService.findAllPosts().stream()
.map(Post::getPostId)
.collect(Collectors.toList())));
}
private void restart(@NonNull List<String> postIds) {
Map<Post, Collection<Image>> data = new HashMap<>();
for (String postId : postIds) {
if (isPending(postId)) {
log.warn(
String.format("Cannot restart, jobs are currently running for post id %s", postIds));
continue;
}
List<Image> images = dataService.findByPostIdAndIsNotCompleted(postId);
if (images.isEmpty()) {
continue;
}
Post post = dataService.findPostByPostId(postId).orElseThrow();
log.debug(String.format("Restarting %d jobs for post id %s", images.size(), postIds));
data.put(post, images);
}
enqueue(data);
}
private synchronized boolean isPending(String postId) {
return pending.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private synchronized boolean isRunning(String postId) {
return running.stream().anyMatch(p -> p.getPost().getPostId().equals(postId));
}
private synchronized void stop(List<String> postIds) {
for (String postId : postIds) {
final Post post = dataService.findPostByPostId(postId).orElseThrow();
if (post == null) {
continue;
}
pending.removeIf(p -> p.getPost().equals(post));
stopRunning(postId);
dataService.stopImagesByPostIdAndIsNotCompleted(postId);
dataService.finishPost(post);
}
metadataService.stopFetchingMetadata(postIds);
}
private boolean canRun(Host host) {
boolean canRun;
int totalRunning = threadCount.values().stream().mapToInt(AtomicInteger::get).sum();
canRun =
threadCount.get(host).get() < settingsService.getSettings().getMaxThreads()
&& (settingsService.getSettings().getMaxTotalThreads() == 0
? totalRunning < MAX_POOL_SIZE
: totalRunning < settingsService.getSettings().getMaxTotalThreads());
if (canRun) {
threadCount.get(host).incrementAndGet();
return true;
}
return false;
}
private Map<Host, Integer> candidateCount() {
HashMap<Host, Integer> map = new HashMap<>();
hosts.forEach(
h -> {
AtomicInteger count = threadCount.get(h);
if (count == null) {
count = new AtomicInteger(0);
threadCount.put(h, count);
}
map.put(h, settingsService.getSettings().getMaxThreads() - count.get());
});
return map;
}
private List<DownloadJob> getCandidates(Map<Host, Integer> candidateCount) {
List<DownloadJob> candidates = new ArrayList<>();
for (DownloadJob downloadJob : pending) {
Host host = downloadJob.getImage().getHost();
Integer maxPerHost = candidateCount.get(host);
if (maxPerHost > 0) {
candidates.add(downloadJob);
candidateCount.put(host, maxPerHost - 1);
}
}
return candidates;
}
public void enqueue(Map<Post, Collection<Image>> images) {
synchronized (this) {
for (Map.Entry<Post, Collection<Image>> entry : images.entrySet()) {
entry.getKey().setStatus(Status.PENDING);
dataService.updatePostStatus(entry.getKey().getStatus(), entry.getKey().getId());
for (Image image : entry.getValue()) {
log.debug(String.format("Enqueuing a job for %s", image.getUrl()));
image.init();
dataService.updateImageStatus(image.getStatus(), image.getId());
dataService.updateImageCurrent(image.getCurrent(), image.getId());
DownloadJob downloadJob =
new DownloadJob(
entry.getKey(), image, (Settings) settingsService.getSettings().clone());
pending.add(downloadJob);
}
}
pending.sort(Comparator.comparing(e -> e.getPost().getAddedOn()));
this.notify();
}
}
private synchronized void schedule(DownloadJob downloadJob) {
log.debug(String.format("Scheduling a job for %s", downloadJob.getImage().getUrl()));
executor.execute(new DownloadJobWrapper(downloadJob));
running.add(downloadJob);
}
public void afterJobFinish(DownloadJob downloadJob) {
synchronized (this) {
running.remove(downloadJob);
threadCount.get(downloadJob.getImage().getHost()).decrementAndGet();
if (!isPending(downloadJob.getPost().getPostId())
&& !isRunning(downloadJob.getPost().getPostId())) {
dataService.finishPost(downloadJob.getPost());
}
this.notify();
}
}
public int pendingCount() {
return pending.size();
}
public int runningCount() {
return running.size();
}
}
@@ -0,0 +1,34 @@
package me.vripper.event;
import lombok.Getter;
@Getter
public class Event<T> {
private final T data;
private final Kind kind;
private Event(Kind kind, T data) {
this.data = data;
this.kind = kind;
}
public static <T> Event<T> wrap(Kind kind, T data) {
return new Event<>(kind, data);
}
public enum Kind {
POST_UPDATE,
POST_REMOVE,
IMAGE_UPDATE,
METADATA_UPDATE,
QUEUED_UPDATE,
QUEUED_REMOVE,
LOG_EVENT_UPDATE,
LOG_EVENT_REMOVE,
VG_USER,
GLOBAL_STATE,
BYTES_PER_SECOND,
SETTINGS_UPDATE
}
}
@@ -0,0 +1,26 @@
package me.vripper.event;
import jakarta.annotation.PreDestroy;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Sinks;
@Service
public class EventBus {
public static final Sinks.EmitFailureHandler RETRY = (signalType, emitResult) -> true;
private final Sinks.Many<Event<?>> sink = Sinks.many().multicast().onBackpressureBuffer();
public void publishEvent(Event<?> event) {
sink.emitNext(event, RETRY);
}
public Flux<Event<?>> flux() {
return sink.asFlux();
}
@PreDestroy
private void destroy() {
sink.emitComplete(RETRY);
}
}
@@ -0,0 +1,12 @@
package me.vripper.exception;
public class DownloadException extends Exception {
public DownloadException(String message) {
super(message);
}
public DownloadException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,15 @@
package me.vripper.exception;
public class HostException extends Exception {
public HostException(Throwable e) {
super(e);
}
public HostException(String message) {
super(message);
}
public HostException(String message, Throwable e) {
super(message, e);
}
}
@@ -0,0 +1,8 @@
package me.vripper.exception;
public class HtmlProcessorException extends Exception {
public HtmlProcessorException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,16 @@
package me.vripper.exception;
public class PostParseException extends Exception {
public PostParseException(String message) {
super(message);
}
public PostParseException(String message, Throwable e) {
super(message, e);
}
public PostParseException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,16 @@
package me.vripper.exception;
public class QueueException extends Exception {
public QueueException(String message) {
super(message);
}
public QueueException(String message, Throwable e) {
super(message, e);
}
public QueueException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,10 @@
package me.vripper.exception;
import java.io.IOException;
public class RenameException extends Exception {
public RenameException(String message, IOException e) {
super(message, e);
}
}
@@ -0,0 +1,8 @@
package me.vripper.exception;
public class ValidationException extends Exception {
public ValidationException(String message) {
super(message);
}
}
@@ -0,0 +1,11 @@
package me.vripper.exception;
public class VripperException extends Exception {
public VripperException(String message) {
super(message);
}
public VripperException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,7 @@
package me.vripper.exception;
public class XpathException extends Exception {
public XpathException(Throwable e) {
super(e);
}
}
@@ -0,0 +1,124 @@
package me.vripper.host;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.ConnectionService;
import me.vripper.services.HostService;
import me.vripper.services.HtmlProcessorService;
import me.vripper.services.XpathService;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class AcidimgHost extends Host {
private static final String host = "acidimg.cc";
private static final String CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']";
private static final String IMG_XPATH = "//img[@class='centred']";
private final ConnectionService cm;
private final HostService hostService;
private final XpathService xpathService;
private final HtmlProcessorService htmlProcessorService;
@Autowired
public AcidimgHost(
ConnectionService cm,
HostService hostService,
XpathService xpathService,
HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.hostService = hostService;
this.xpathService = xpathService;
this.htmlProcessorService = htmlProcessorService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node contDiv;
try {
log.debug(
String.format(
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, image.getUrl()));
contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (contDiv != null) {
log.debug(String.format("Click button found for %s", image.getUrl()));
HttpClient client = cm.getClient().build();
HttpPost httpPost = cm.buildHttpPost(image.getUrl(), context);
httpPost.addHeader("Referer", image.getUrl());
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("imgContinue", "Continue to your image"));
try {
httpPost.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
throw new HostException(e);
}
log.debug(String.format("Requesting %s", httpPost));
try (CloseableHttpResponse response =
(CloseableHttpResponse) client.execute(httpPost, context)) {
log.debug(String.format("Cleaning response for %s", httpPost));
doc = htmlProcessorService.clean(EntityUtils.toString(response.getEntity()));
EntityUtils.consumeQuietly(response.getEntity());
} catch (Exception e) {
throw new HostException(e);
}
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Cannot find the image node");
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? hostService.getDefaultImageName(imgUrl) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,86 @@
package me.vripper.host;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class DPicMeHost extends Host {
private static final String IMG_XPATH = "//img[@id='pic']";
private static final String host = "dpic.me";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public DPicMeHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
HostService.Response response =
hostService.getResponse(image.getUrl().replace("http://", "https://"), context);
Document doc = response.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException(
String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.getUrl()));
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(e -> e.getTextContent().trim())
.orElse("");
String imgUrl =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("src"))
.map(e -> e.getTextContent().trim())
.orElse("");
String defaultName = UUID.randomUUID().toString();
int index = imgUrl.lastIndexOf('/');
if (index != -1 && index < imgUrl.length()) {
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
}
return new HostService.NameUrl(imgTitle.isEmpty() ? defaultName : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,43 @@
package me.vripper.host;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public abstract class Host {
public abstract String getHost();
public abstract String getLookup();
public boolean isSupported(String url) {
return url.contains(getLookup());
}
public abstract HostService.NameUrl getNameAndUrl(
final Image image, final HttpClientContext context) throws HostException;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Host host = (Host) o;
return Objects.equals(getHost(), host.getHost());
}
@Override
public int hashCode() {
return Objects.hash(getHost());
}
@Override
public String toString() {
return getHost();
}
}
@@ -0,0 +1,104 @@
package me.vripper.host;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageBamHost extends Host {
private static final String host = "imagebam.com";
private static final String IMG_XPATH = "//img[contains(@class,'main-image')]";
private static final String CONTINUE_XPATH = "//*[contains(text(), 'Continue')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageBamHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
context.setCookieStore(new BasicCookieStore());
BasicClientCookie cookie = new BasicClientCookie("nsfw_inter", "1");
cookie.setDomain("www.imagebam.com");
context.getCookieStore().addCookie(cookie);
HostService.Response response = hostService.getResponse(image.getUrl(), context);
Document doc = response.getDocument();
try {
log.debug(
String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, image.getUrl()));
if (xpathService.getAsNode(doc, CONTINUE_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
response = hostService.getResponse(image.getUrl(), context);
doc = response.getDocument();
}
} catch (XpathException e) {
throw new HostException(e);
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException(
String.format("Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.getUrl()));
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(e -> e.getTextContent().trim())
.orElse("");
String imgUrl =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("src"))
.map(e -> e.getTextContent().trim())
.orElse("");
String defaultName = UUID.randomUUID().toString();
int index = imgUrl.lastIndexOf('/');
if (index != -1 && index < imgUrl.length()) {
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
}
return new HostService.NameUrl(imgTitle.isEmpty() ? defaultName : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageTwistHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "imagetwist.com";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageTwistHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(Node::getTextContent)
.map(String::trim)
.orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,85 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageVenueHost extends Host {
private static final String host = "imagevenue.com";
private static final String CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']";
private static final String IMG_XPATH = "//a[@data-toggle='full']/img";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageVenueHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
if (xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
resp = hostService.getResponse(url, context);
doc = resp.getDocument();
}
} catch (XpathException e) {
throw new HostException(e);
}
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,72 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImageZillaHost extends Host {
private static final String host = "imagezilla.net";
private static final String lookup = "imagezilla.net/show";
private static final String IMG_XPATH = "//img[@id='photo']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImageZillaHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return lookup;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
String title;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
Node titleNode = xpathService.getAsNode(doc, IMG_XPATH).getAttributes().getNamedItem("title");
log.debug(String.format("Resolving name for %s", image.getUrl()));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
}
} catch (XpathException e) {
throw new HostException(e);
}
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(image.getUrl());
}
try {
return new HostService.NameUrl(title, image.getUrl().replace("show", "images"));
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,68 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImgSpiceHost extends Host {
private static final String host = "imgspice.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgSpiceHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,65 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ImgboxHost extends Host {
private static final String host = "imgbox.com";
private static final String IMG_XPATH = "//img[@id='img']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ImgboxHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("title").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,40 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class ImxHost extends Host {
private static final String host = "imx.to";
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = String.format("IMG_%04d", image.getIndex() + 1);
String imgUrl =
image
.getThumbUrl()
.replace("http:", "https:")
.replace("upload/small/", "u/i/")
.replace("u/t/", "u/i/");
return new HostService.NameUrl(imgTitle, imgUrl);
}
}
@@ -0,0 +1,76 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PimpandhostHost extends Host {
private static final String host = "pimpandhost.com";
private static final String IMG_XPATH = "//img[contains(@class, 'original')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PimpandhostHost(HostService hostService, XpathService xpathService) {
super();
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url;
try {
url = hostService.appendUri(image.getUrl().replace("http://", "https://"), "size=original");
} catch (Exception e) {
throw new HostException(e);
}
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl =
"https:" + imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PixRouteHost extends Host {
private static final String host = "pixroute.com";
private static final String IMG_XPATH = "//img[@id='imgpreview']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixRouteHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
if (imgNode == null) {
throw new HostException("Failed to locate image");
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
return new HostService.NameUrl(
imgNode.getAttributes().getNamedItem("alt").getTextContent().trim(),
imgNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,66 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PixhostHost extends Host {
private static final String host = "pixhost.to";
private static final String lookup = "pixhost.to/show";
private static final String IMG_XPATH = "//img[@id='image']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixhostHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return lookup;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle = imgNode.getAttributes().getNamedItem("alt").getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,73 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PixxxelsHost extends Host {
private static final String host = "pixxxels.cc";
private static final String IMG_XPATH = "//*[@id='download']";
private static final String TITLE_XPATH = "//*[contains(@class,'imagename')]";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PixxxelsHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
HostService.Response resp = hostService.getResponse(url, context);
Document doc = resp.getDocument();
Node imgNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle = titleNode.getTextContent().trim();
String imgUrl = imgNode.getAttributes().getNamedItem("href").getTextContent().trim();
return new HostService.NameUrl(
imgTitle.isEmpty() ? imgUrl.substring(imgUrl.lastIndexOf('/') + 1) : imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,74 @@
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class PostImgHost extends Host {
private static final String host = "postimg.cc";
private static final String TITLE_XPATH = "//span[contains(@class,'imagename')]";
private static final String IMG_XPATH = "//a[@id='download']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public PostImgHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
String url = image.getUrl().replace("http://", "https://");
Document doc = hostService.getResponse(url, context).getDocument();
Node urlNode, titleNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url));
titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url));
urlNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", url));
String imgTitle =
Optional.ofNullable(titleNode)
.map(node -> node.getTextContent().trim())
.orElseGet(() -> hostService.getDefaultImageName(url));
return new HostService.NameUrl(
imgTitle, urlNode.getAttributes().getNamedItem("href").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,75 @@
package me.vripper.host;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class TurboImageHost extends Host {
private static final String host = "turboimagehost.com";
private static final String TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1";
private static final String IMG_XPATH = "//img[@id='imageid']";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public TurboImageHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
String title;
try {
log.debug(
String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.getUrl()));
Node titleNode = xpathService.getAsNode(doc, TITLE_XPATH);
log.debug(String.format("Resolving name for %s", image.getUrl()));
if (titleNode != null) {
title = titleNode.getTextContent().trim();
} else {
title = null;
}
} catch (XpathException e) {
throw new HostException(e);
}
if (title == null || title.isEmpty()) {
title = hostService.getDefaultImageName(image.getUrl());
}
try {
Node urlNode = xpathService.getAsNode(doc, IMG_XPATH);
return new HostService.NameUrl(
title, urlNode.getAttributes().getNamedItem("src").getTextContent().trim());
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,70 @@
package me.vripper.host;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.XpathException;
import me.vripper.jpa.domain.Image;
import me.vripper.services.HostService;
import me.vripper.services.XpathService;
import org.apache.http.client.protocol.HttpClientContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@Service
@Slf4j
public class ViprImHost extends Host {
private static final String IMG_XPATH = "//img[contains(@class, 'img')]";
private static final String host = "vipr.im";
private final HostService hostService;
private final XpathService xpathService;
@Autowired
public ViprImHost(HostService hostService, XpathService xpathService) {
this.hostService = hostService;
this.xpathService = xpathService;
}
@Override
public String getHost() {
return host;
}
@Override
public String getLookup() {
return host;
}
@Override
public HostService.NameUrl getNameAndUrl(final Image image, final HttpClientContext context)
throws HostException {
Document doc = hostService.getResponse(image.getUrl(), context).getDocument();
Node imgNode;
try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.getUrl()));
imgNode = xpathService.getAsNode(doc, IMG_XPATH);
} catch (XpathException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Resolving name and image url for %s", image.getUrl()));
String imgTitle =
Optional.ofNullable(imgNode.getAttributes().getNamedItem("alt"))
.map(Node::getTextContent)
.map(String::trim)
.orElse(null);
String imgUrl = imgNode.getAttributes().getNamedItem("src").getTextContent().trim();
return new HostService.NameUrl(imgTitle, imgUrl);
} catch (Exception e) {
throw new HostException("Unexpected error occurred", e);
}
}
}
@@ -0,0 +1,78 @@
package me.vripper.jpa;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.io.File;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import me.vripper.download.DownloadService;
import me.vripper.services.ThreadPoolService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Component
@EnableScheduling
public class Management {
public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Pattern BACKUP_FILE_PATTERN =
Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$");
private final JdbcTemplate jdbcTemplate;
private final String backupFolder;
private final ThreadPoolService threadPoolService;
private final DownloadService downloadService;
public Management(
JdbcTemplate jdbcTemplate,
@Value("${base.dir}") String baseDir,
@Value("${base.dir.name}") String baseDirName,
ThreadPoolService threadPoolService,
DownloadService downloadService) {
this.jdbcTemplate = jdbcTemplate;
this.threadPoolService = threadPoolService;
this.downloadService = downloadService;
this.backupFolder = baseDir + File.separator + baseDirName + File.separator + "backup";
}
@PreDestroy
public void destroy() {
try {
threadPoolService.destroy();
downloadService.destroy();
this.jdbcTemplate.execute("SHUTDOWN");
} catch (Exception ignored) {
Thread.currentThread().interrupt();
}
}
@PostConstruct
@Scheduled(cron = "0 0 0 ? * *")
private void backup() {
for (File file :
Optional.ofNullable(new File(backupFolder).listFiles()).orElse(new File[] {})) {
Matcher matcher = BACKUP_FILE_PATTERN.matcher(file.getName());
if (matcher.find()) {
LocalDate localDate = LocalDate.parse(matcher.group(1), FORMATTER);
if (localDate.isEqual(LocalDate.now())) {
return;
}
}
}
// create a backup file
File backupFile =
new File(
backupFolder, String.format("db_%s.tar.gz", LocalDateTime.now().format(FORMATTER)));
jdbcTemplate.execute(String.format("BACKUP DATABASE TO '%s' BLOCKING", backupFile.getPath()));
}
}
@@ -0,0 +1,24 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
public class DateTimeSerializer extends StdSerializer<LocalDateTime> {
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
protected DateTimeSerializer() {
super(LocalDateTime.class);
}
@Override
public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider)
throws IOException {
gen.writeString(value.format(DATE_TIME_FORMATTER));
}
}
@@ -0,0 +1,68 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import me.vripper.host.Host;
import me.vripper.jpa.domain.enums.Status;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Image {
@JsonIgnore protected Long id;
@JsonIgnore private Host host;
private String url;
private String thumbUrl;
private int index;
private long current = 0;
private long total = 0;
private Status status;
private String postId;
@JsonIgnore private Long postIdRef;
public Image(String postId, String url, String thumbUrl, Host host, int index) {
this.postId = postId;
this.url = url;
this.thumbUrl = thumbUrl;
this.host = host;
this.index = index;
status = Status.STOPPED;
}
public void increase(int read) {
current += read;
}
public void init() {
current = 0;
status = Status.STOPPED;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Image image = (Image) o;
return Objects.equals(url, image.url);
}
@Override
public int hashCode() {
return Objects.hash(url);
}
}
@@ -0,0 +1,51 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.time.LocalDateTime;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class LogEvent {
private Long id;
private Type type;
private Status status;
@JsonSerialize(using = DateTimeSerializer.class)
private LocalDateTime time;
private String message;
public LogEvent(Type type, Status status, LocalDateTime time, String message) {
this.type = type;
this.status = status;
this.time = time;
this.message = message;
}
public enum Type {
POST,
QUEUED,
THANKS,
METADATA,
SCAN,
DOWNLOAD,
// Below are deprecated events
METADATA_CACHE_MISS,
QUEUED_CACHE_MISS
}
public enum Status {
PENDING,
PROCESSING,
DONE,
ERROR
}
}
@@ -0,0 +1,30 @@
package me.vripper.jpa.domain;
import java.util.Collections;
import java.util.List;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@NoArgsConstructor
public class Metadata {
private Long postIdRef;
private String postId;
private String postedBy;
private List<String> resolvedNames = Collections.emptyList();
public static Metadata from(Metadata metadata) {
Metadata copy = new Metadata();
copy.postIdRef = metadata.postIdRef;
copy.postId = metadata.postId;
copy.postedBy = metadata.postedBy;
copy.resolvedNames = List.copyOf(metadata.resolvedNames);
return copy;
}
}
@@ -0,0 +1,89 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import me.vripper.jpa.domain.enums.Status;
@Getter
@Setter
@ToString
@NoArgsConstructor
public class Post {
@JsonIgnore private Long id;
private Status status;
private String postId;
private String threadTitle;
private String threadId;
private String title;
private String url;
private int done;
private int total;
private Set<String> hosts;
private String forum;
@JsonIgnore private String securityToken;
@JsonIgnore private String downloadDirectory;
private boolean thanked;
private Set<String> previews = Collections.emptySet();
private Metadata metadata;
private int rank = Integer.MAX_VALUE;
@JsonSerialize(using = DateTimeSerializer.class)
private LocalDateTime addedOn;
public Post(
String title,
String url,
String postId,
String threadId,
String threadTitle,
String forum,
String securityToken) {
this.title = title;
this.url = url;
this.postId = postId;
this.forum = forum;
this.threadId = threadId;
this.threadTitle = threadTitle;
this.securityToken = securityToken;
status = Status.STOPPED;
addedOn = LocalDateTime.now();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Post post = (Post) o;
return Objects.equals(postId, post.postId);
}
@Override
public int hashCode() {
return Objects.hash(postId);
}
}
@@ -0,0 +1,55 @@
package me.vripper.jpa.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@NoArgsConstructor
@ToString
public class Queued {
@JsonIgnore private Long id;
private String link;
private String threadId;
private String postId;
private int total = 0;
private boolean loading = true;
public Queued(String link, String threadId, String postId) {
this();
this.link = link;
this.threadId = threadId;
this.postId = postId;
}
public void done() {
this.loading = false;
}
public void increment() {
this.total++;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Queued that = (Queued) o;
return Objects.equals(threadId, that.threadId);
}
@Override
public int hashCode() {
return Objects.hash(threadId);
}
}
@@ -0,0 +1,10 @@
package me.vripper.jpa.domain.enums;
public enum Status {
PENDING,
DOWNLOADING,
COMPLETE,
ERROR,
STOPPED,
PARTIAL
}
@@ -0,0 +1,31 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.enums.Status;
public interface IImageRepository extends IRepository {
Image save(Image image);
void deleteAllByPostId(String postId);
List<Image> findByPostId(String postId);
Integer countError();
List<Image> findByPostIdAndIsNotCompleted(String postId);
int stopByPostIdAndIsNotCompleted(String postId);
List<Image> findByPostIdAndIsError(String postId);
Optional<Image> findById(Long id);
int updateStatus(Status status, Long id);
int updateCurrent(long current, Long id);
int updateTotal(long total, Long id);
}
@@ -0,0 +1,20 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.LogEvent;
public interface ILogEventRepository extends IRepository {
LogEvent save(LogEvent logEvent);
LogEvent update(LogEvent logEvent);
List<LogEvent> findAll();
Optional<LogEvent> findById(Long id);
void delete(Long id);
void deleteAll();
}
@@ -0,0 +1,13 @@
package me.vripper.jpa.repositories;
import java.util.Optional;
import me.vripper.jpa.domain.Metadata;
public interface IMetadataRepository extends IRepository {
Metadata save(Metadata metadata);
Optional<Metadata> findByPostId(String postId);
int deleteByPostId(String postId);
}
@@ -0,0 +1,37 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
public interface IPostRepository extends IRepository {
Post save(Post post);
Optional<Post> findByPostId(String postId);
Optional<Post> findById(Long id);
List<String> findCompleted();
List<Post> findAll();
boolean existByPostId(String postId);
int setDownloadingToStopped();
int deleteByPostId(String postId);
int updateStatus(Status status, Long id);
int updateDone(int done, Long id);
int updateFolderName(String postFolderName, Long id);
int updateTitle(String title, Long id);
int updateThanked(boolean thanked, Long id);
int updateRank(int rank, Long id);
}
@@ -0,0 +1,19 @@
package me.vripper.jpa.repositories;
import java.util.List;
import java.util.Optional;
import me.vripper.jpa.domain.Queued;
public interface IQueuedRepository extends IRepository {
Queued save(Queued queued);
Optional<Queued> findByThreadId(String threadId);
List<Queued> findAll();
Optional<Queued> findById(Long id);
int deleteByThreadId(String threadId);
void deleteAll();
}
@@ -0,0 +1,3 @@
package me.vripper.jpa.repositories;
public interface IRepository {}
@@ -0,0 +1,157 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.SpringContext;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IImageRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class ImageRepository implements IImageRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public ImageRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_IMAGE", Long.class);
}
@Override
public Image save(Image image) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, THUMB_URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?,?)",
id,
image.getCurrent(),
image.getHost().getHost(),
image.getIndex(),
image.getPostId(),
image.getStatus().name(),
image.getTotal(),
image.getUrl(),
image.getThumbUrl(),
image.getPostIdRef());
image.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return image;
}
@Override
public void deleteAllByPostId(String postId) {
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId);
}
@Override
public List<Image> findByPostId(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE WHERE POST_ID = ?", new ImageRowMapper(), postId);
}
@Override
public Integer countError() {
return jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = 'ERROR'", Integer.class);
}
@Override
public List<Image> findByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
new ImageRowMapper(),
postId);
}
@Override
public int stopByPostIdAndIsNotCompleted(String postId) {
return jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = 'STOPPED' WHERE image.POST_ID = ? AND image.STATUS <> 'COMPLETE'",
postId);
}
@Override
public List<Image> findByPostIdAndIsError(String postId) {
return jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = 'ERROR'",
new ImageRowMapper(),
postId);
}
@Override
public Optional<Image> findById(Long id) {
List<Image> images =
jdbcTemplate.query(
"SELECT * FROM IMAGE AS image WHERE image.ID = ?", new ImageRowMapper(), id);
if (images.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(images.get(0));
}
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.ID = ?", status.name(), id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return mutationCount;
}
@Override
public int updateCurrent(long current, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.CURRENT = ? WHERE image.ID = ?", current, id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return mutationCount;
}
@Override
public int updateTotal(long total, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE IMAGE AS image SET image.TOTAL = ? WHERE image.ID = ?", total, id);
eventBus.publishEvent(Event.wrap(Event.Kind.IMAGE_UPDATE, id));
return mutationCount;
}
}
class ImageRowMapper implements RowMapper<Image> {
@Override
public Image mapRow(ResultSet rs, int rowNum) throws SQLException {
Image image = new Image();
image.setId(rs.getLong("ID"));
String host = rs.getString("HOST");
image.setHost(
SpringContext.getBeansOfType(Host.class).values().stream()
.filter(e -> e.getHost().equals(host))
.findAny()
.orElse(null));
image.setUrl(rs.getString("URL"));
image.setThumbUrl(rs.getString("THUMB_URL"));
image.setIndex(rs.getInt("INDEX"));
image.setCurrent(rs.getLong("CURRENT"));
image.setTotal(rs.getLong("TOTAL"));
image.setStatus(Status.valueOf(rs.getString("STATUS")));
image.setPostId(rs.getString("POST_ID"));
image.setPostIdRef(rs.getLong("POST_ID_REF"));
return image;
}
}
@@ -0,0 +1,124 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.LogEvent;
import me.vripper.jpa.repositories.ILogEventRepository;
import me.vripper.services.SettingsService;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class LogEventRepository implements ILogEventRepository {
private final JdbcTemplate jdbcTemplate;
private final SettingsService settingsService;
private final EventBus eventBus;
public LogEventRepository(
JdbcTemplate jdbcTemplate, SettingsService settingsService, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.settingsService = settingsService;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_EVENT", Long.class);
}
@Override
public synchronized LogEvent save(@NonNull LogEvent logEvent) {
int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
if (count > maxRecords) {
List<Long> idList =
jdbcTemplate.queryForList(
"SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
idList.forEach(this::delete);
}
long id = nextId();
jdbcTemplate.update(
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
id,
logEvent.getType().name(),
logEvent.getStatus().name(),
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
logEvent.getMessage());
logEvent.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, id));
return logEvent;
}
@Override
public LogEvent update(@NonNull LogEvent logEvent) {
if (logEvent.getId() == null) {
log.warn("Cannot update entity with null id");
return logEvent;
}
jdbcTemplate.update(
"UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
logEvent.getType().name(),
logEvent.getStatus().name(),
logEvent.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
logEvent.getMessage(),
logEvent.getId());
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_UPDATE, logEvent.getId()));
return logEvent;
}
@Override
public Optional<LogEvent> findById(Long id) {
List<LogEvent> logEvents =
jdbcTemplate.query("SELECT * FROM EVENT WHERE ID = ?", new LogEventRowMapper(), id);
if (logEvents.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(logEvents.get(0));
}
}
@Override
public List<LogEvent> findAll() {
return jdbcTemplate.query("SELECT * FROM EVENT", new LogEventRowMapper());
}
@Override
public void delete(Long id) {
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
eventBus.publishEvent(Event.wrap(Event.Kind.LOG_EVENT_REMOVE, id));
}
@Override
public void deleteAll() {
jdbcTemplate.update("DELETE FROM EVENT");
}
}
class LogEventRowMapper implements RowMapper<LogEvent> {
@Override
public LogEvent mapRow(ResultSet rs, int rowNum) throws SQLException {
LogEvent logEvent = new LogEvent();
logEvent.setId(rs.getLong("ID"));
logEvent.setType(LogEvent.Type.valueOf(rs.getString("TYPE")));
logEvent.setStatus(LogEvent.Status.valueOf(rs.getString("STATUS")));
logEvent.setTime(
LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
logEvent.setMessage(rs.getString("MESSAGE"));
return logEvent;
}
}
@@ -0,0 +1,75 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.repositories.IMetadataRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class MetadataRepository implements IMetadataRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public MetadataRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
@Override
public Metadata save(Metadata metadata) {
jdbcTemplate.update(
"INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
metadata.getPostIdRef(),
metadata.getPostId(),
metadata.getPostedBy(),
String.join("%sep%", metadata.getResolvedNames()));
eventBus.publishEvent(Event.wrap(Event.Kind.METADATA_UPDATE, metadata.getPostIdRef()));
return metadata;
}
@Override
public Optional<Metadata> findByPostId(String postId) {
List<Metadata> metadata =
jdbcTemplate.query(
"SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
new MetadataRowMapper(),
postId);
if (metadata.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(metadata.get(0));
}
}
@Override
public int deleteByPostId(String postId) {
return jdbcTemplate.update(
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?", postId);
}
}
class MetadataRowMapper implements RowMapper<Metadata> {
@Override
public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
Metadata metadata = new Metadata();
metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
metadata.setPostId(rs.getString("POST_ID"));
metadata.setPostedBy(rs.getString("POSTED_BY"));
String resolvedNames = rs.getString("RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
return metadata;
}
}
@@ -0,0 +1,229 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Metadata;
import me.vripper.jpa.domain.Post;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IPostRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class PostRepository implements IPostRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public PostRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_POST", Long.class);
}
@Override
public Post save(Post post) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL, ADDED_ON, RANK) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
id,
post.getDone(),
post.getForum(),
String.join(";", post.getHosts()),
post.getDownloadDirectory(),
post.getPostId(),
String.join(";", post.getPreviews()),
post.getSecurityToken(),
post.getStatus().name(),
post.isThanked(),
post.getThreadId(),
post.getThreadTitle(),
post.getTitle(),
post.getTotal(),
post.getUrl(),
Timestamp.valueOf(post.getAddedOn()),
post.getRank());
post.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return post;
}
@Override
public Optional<Post> findByPostId(String postId) {
List<Post> posts =
jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
new PostRowMapper(),
postId);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<String> findCompleted() {
return jdbcTemplate.query(
"SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total",
((rs, rowNum) -> rs.getString("POST_ID")));
}
@Override
public Optional<Post> findById(Long id) {
List<Post> posts =
jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
new PostRowMapper(),
id);
if (posts.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(posts.get(0));
}
}
@Override
public List<Post> findAll() {
return jdbcTemplate.query(
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
new PostRowMapper());
}
@Override
public boolean existByPostId(String postId) {
Integer count =
jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?", Integer.class, postId);
if (count == null) {
return false;
} else {
return count > 0;
}
}
@Override
public int setDownloadingToStopped() {
return jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = 'STOPPED' WHERE post.STATUS = 'DOWNLOADING' OR post.STATUS = 'PARTIAL' OR post.STATUS = 'PENDING'");
}
@Override
public int deleteByPostId(String postId) {
int mutationCount =
jdbcTemplate.update("DELETE FROM POST AS post WHERE post.POST_ID = ?", postId);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_REMOVE, postId));
return mutationCount;
}
@Override
public int updateStatus(Status status, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE POST AS post SET post.STATUS = ? WHERE post.ID = ?", status.name(), id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateDone(int done, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.DONE = ? WHERE post.ID = ?", done, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateFolderName(String postFolderName, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE POST AS post SET post.POST_FOLDER_NAME = ? WHERE post.ID = ?",
postFolderName,
id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateTitle(String title, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.TITLE = ? WHERE post.ID = ?", title, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateThanked(boolean thanked, Long id) {
int mutationCount =
jdbcTemplate.update(
"UPDATE POST AS post SET post.THANKED = ? WHERE post.ID = ?", thanked, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
@Override
public int updateRank(int rank, Long id) {
int mutationCount =
jdbcTemplate.update("UPDATE POST AS post SET post.RANK = ? WHERE post.ID = ?", rank, id);
eventBus.publishEvent(Event.wrap(Event.Kind.POST_UPDATE, id));
return mutationCount;
}
}
class PostRowMapper implements RowMapper<Post> {
private static final String DELIMITER = ";";
@Override
public Post mapRow(ResultSet rs, int rowNum) throws SQLException {
Post post = new Post();
post.setId(rs.getLong("post.ID"));
post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
post.setPostId(rs.getString("post.POST_ID"));
post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
post.setThreadId(rs.getString("post.THREAD_ID"));
post.setTitle(rs.getString("post.TITLE"));
post.setUrl(rs.getString("post.URL"));
post.setDone(rs.getInt("post.DONE"));
post.setTotal(rs.getInt("post.TOTAL"));
post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
post.setForum(rs.getString("post.FORUM"));
post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
post.setThanked(rs.getBoolean("post.THANKED"));
String previews;
if ((previews = rs.getString("post.PREVIEWS")) != null) {
post.setPreviews(Set.of(previews.split(DELIMITER)));
}
post.setAddedOn(rs.getTimestamp("post.ADDED_ON").toLocalDateTime());
post.setRank(rs.getInt("post.RANK"));
Long metadataId = rs.getLong("metadata.POST_ID_REF");
if (!rs.wasNull()) {
Metadata metadata = new Metadata();
metadata.setPostIdRef(metadataId);
metadata.setPostId(rs.getString("metadata.POST_ID"));
String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
if (resolvedNames != null && !resolvedNames.isBlank()) {
metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
}
metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
post.setMetadata(metadata);
}
return post;
}
}
@@ -0,0 +1,106 @@
package me.vripper.jpa.repositories.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Queued;
import me.vripper.jpa.repositories.IQueuedRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
@Service
public class QueuedRepository implements IQueuedRepository {
private final JdbcTemplate jdbcTemplate;
private final EventBus eventBus;
@Autowired
public QueuedRepository(JdbcTemplate jdbcTemplate, EventBus eventBus) {
this.jdbcTemplate = jdbcTemplate;
this.eventBus = eventBus;
}
private synchronized Long nextId() {
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_QUEUED", Long.class);
}
@Override
public Queued save(Queued queued) {
long id = nextId();
jdbcTemplate.update(
"INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
id,
queued.getTotal(),
queued.getLink(),
queued.isLoading(),
queued.getPostId(),
queued.getThreadId());
queued.setId(id);
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_UPDATE, id));
return queued;
}
@Override
public Optional<Queued> findByThreadId(String threadId) {
List<Queued> queuedList =
jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.THREAD_ID = ?",
new QueuedRowMapper(),
threadId);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public List<Queued> findAll() {
return jdbcTemplate.query("SELECT * FROM QUEUED", new QueuedRowMapper());
}
@Override
public Optional<Queued> findById(Long id) {
List<Queued> queuedList =
jdbcTemplate.query(
"SELECT * FROM QUEUED AS queued WHERE queued.ID = ?", new QueuedRowMapper(), id);
if (queuedList.isEmpty()) {
return Optional.empty();
} else {
return Optional.of(queuedList.get(0));
}
}
@Override
public int deleteByThreadId(String threadId) {
int mutationCount =
jdbcTemplate.update("DELETE FROM QUEUED AS queued WHERE THREAD_ID = ?", threadId);
eventBus.publishEvent(Event.wrap(Event.Kind.QUEUED_REMOVE, threadId));
return mutationCount;
}
@Override
public void deleteAll() {
jdbcTemplate.update("DELETE FROM QUEUED");
}
}
class QueuedRowMapper implements RowMapper<Queued> {
@Override
public Queued mapRow(ResultSet rs, int rowNum) throws SQLException {
Queued queued = new Queued();
queued.setId(rs.getLong("ID"));
queued.setLink(rs.getString("LINK"));
queued.setThreadId(rs.getString("THREAD_ID"));
queued.setPostId(rs.getString("POST_ID"));
queued.setTotal(rs.getInt("TOTAL"));
queued.setLoading(rs.getBoolean("LOADING"));
return queued;
}
}
@@ -0,0 +1,153 @@
package me.vripper.services;
import jakarta.annotation.PreDestroy;
import java.net.URI;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.download.DownloadJob;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.services.domain.Settings;
import net.jodah.failsafe.RetryPolicy;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.AbstractExecutionAwareRequest;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@EnableScheduling
@Slf4j
public class ConnectionService {
private final Disposable disposable;
private PoolingHttpClientConnectionManager pcm;
private RequestConfig rc;
@Getter private RetryPolicy<Object> retryPolicy;
private int connectionTimeout;
private int maxAttempts;
public ConnectionService(EventBus eventBus, SettingsService settingsService) {
connectionTimeout = settingsService.getSettings().getConnectionTimeout();
maxAttempts = settingsService.getSettings().getMaxAttempts();
disposable =
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.map(e -> ((Settings) e.getData()))
.subscribe(
settings -> {
if (connectionTimeout != settings.getConnectionTimeout()) {
connectionTimeout = settings.getConnectionTimeout();
buildRequestConfig();
}
if (maxAttempts != settings.getMaxAttempts()) {
maxAttempts = settings.getMaxAttempts();
buildRetryPolicy();
}
});
buildRequestConfig();
buildRetryPolicy();
buildConnectionPool();
}
@PreDestroy
private void destroy() {
disposable.dispose();
}
private void buildRetryPolicy() {
retryPolicy =
new RetryPolicy<>()
.withDelay(2, 5, ChronoUnit.SECONDS)
.withMaxAttempts(maxAttempts)
.onFailedAttempt(
e ->
log.warn(
String.format("#%d tries failed", e.getAttemptCount()),
e.getLastFailure()));
}
private void buildConnectionPool() {
pcm = new PoolingHttpClientConnectionManager();
pcm.setMaxTotal(Integer.MAX_VALUE);
pcm.setDefaultMaxPerRoute(Integer.MAX_VALUE);
}
private void buildRequestConfig() {
rc =
RequestConfig.custom()
.setConnectionRequestTimeout(connectionTimeout * 1000)
.setConnectTimeout(connectionTimeout * 1000)
.setSocketTimeout(connectionTimeout * 1000)
.setCookieSpec(CookieSpecs.STANDARD)
.build();
}
@Scheduled(fixedDelay = 5_000)
private void idleConnectionMonitoring() {
pcm.closeExpiredConnections();
pcm.closeIdleConnections(30, TimeUnit.SECONDS);
}
public HttpClientBuilder getClient() {
return HttpClients.custom()
.setConnectionManager(pcm)
.setRedirectStrategy(new LaxRedirectStrategy())
.disableAutomaticRetries()
.setDefaultRequestConfig(rc);
}
public HttpGet buildHttpGet(String url, final HttpClientContext context) {
HttpGet httpGet = new HttpGet(url.replace(" ", "+"));
httpGet.addHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpGet);
return httpGet;
}
public HttpPost buildHttpPost(String url, final HttpClientContext context) {
HttpPost httpPost = new HttpPost(url.replace(" ", "+"));
httpPost.addHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpPost);
return httpPost;
}
public HttpGet buildHttpGet(URI uri, final HttpClientContext context) {
HttpGet httpGet = new HttpGet(uri);
httpGet.addHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
addToContext(context, httpGet);
return httpGet;
}
public void addToContext(HttpClientContext context, AbstractExecutionAwareRequest request) {
if (context != null) {
List<AbstractExecutionAwareRequest> requests =
(List<AbstractExecutionAwareRequest>)
context.getAttribute(DownloadJob.ContextAttributes.OPEN_CONNECTION.toString());
if (requests != null) {
requests.add(request);
}
}
}
}
@@ -0,0 +1,242 @@
package me.vripper.services;
import java.util.*;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.domain.*;
import me.vripper.jpa.domain.enums.Status;
import me.vripper.jpa.repositories.IImageRepository;
import me.vripper.jpa.repositories.IMetadataRepository;
import me.vripper.jpa.repositories.IPostRepository;
import me.vripper.jpa.repositories.IQueuedRepository;
import me.vripper.jpa.repositories.impl.LogEventRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@Transactional
@Slf4j
public class DataService {
private final IPostRepository postRepository;
private final IImageRepository imageRepository;
private final IQueuedRepository queuedRepository;
private final IMetadataRepository metadataRepository;
private final SettingsService settingsService;
private final LogEventRepository eventRepository;
@Autowired
public DataService(
IPostRepository postRepository,
IImageRepository imageRepository,
IQueuedRepository queuedRepository,
IMetadataRepository metadataRepository,
SettingsService settingsService,
LogEventRepository eventRepository) {
this.postRepository = postRepository;
this.imageRepository = imageRepository;
this.queuedRepository = queuedRepository;
this.metadataRepository = metadataRepository;
this.settingsService = settingsService;
this.eventRepository = eventRepository;
}
private void save(Post post) {
postRepository.save(post);
}
private void save(Queued queued) {
queuedRepository.save(queued);
}
private void save(Image image) {
imageRepository.save(image);
}
public boolean exists(String postId) {
return postRepository.existByPostId(postId);
}
public void newPost(Post post, Collection<Image> images) {
save(post);
images.forEach(
image -> {
image.setPostIdRef(post.getId());
save(image);
});
sortPostsByRank();
}
public synchronized void afterJobFinish(Image image, Post post) {
if (image.getStatus().equals(Status.COMPLETE)) {
post.setDone(post.getDone() + 1);
updatePostDone(post.getDone(), post.getId());
} else if (image.getStatus().equals(Status.ERROR)) {
post.setStatus(Status.PARTIAL);
updatePostStatus(post.getStatus(), post.getId());
}
}
private void updatePostDone(int done, Long id) {
postRepository.updateDone(done, id);
}
public void updatePostStatus(Status status, Long id) {
postRepository.updateStatus(status, id);
}
public void updatePostThanks(boolean thanked, Long id) {
postRepository.updateThanked(thanked, id);
}
public void updatePostTitle(String title, Long id) {
postRepository.updateTitle(title, id);
}
public void updatePostDownloadDirectory(String postFolderName, Long id) {
postRepository.updateFolderName(postFolderName, id);
}
public void updatePostRank(int rank, Long id) {
postRepository.updateRank(rank, id);
}
public void finishPost(@NonNull Post post) {
if (!imageRepository.findByPostIdAndIsError(post.getPostId()).isEmpty()) {
post.setStatus(Status.ERROR);
updatePostStatus(post.getStatus(), post.getId());
} else {
if (post.getDone() < post.getTotal()) {
post.setStatus(Status.STOPPED);
updatePostStatus(post.getStatus(), post.getId());
} else {
post.setStatus(Status.COMPLETE);
updatePostStatus(post.getStatus(), post.getId());
if (settingsService.getSettings().getClearCompleted()) {
remove(List.of(post.getPostId()));
}
}
}
}
private void remove(@NonNull final List<String> postIds) {
for (String postId : postIds) {
imageRepository.deleteAllByPostId(postId);
metadataRepository.deleteByPostId(postId);
postRepository.deleteByPostId(postId);
}
sortPostsByRank();
}
public void newQueueLink(@NonNull final Queued queued) {
save(queued);
}
public void removeQueueLink(@NonNull final String threadId) {
queuedRepository.deleteByThreadId(threadId);
}
public List<String> clearCompleted() {
List<String> completed = postRepository.findCompleted();
remove(completed);
return completed;
}
public void removeAll(final List<String> postIds) {
remove(
Objects.requireNonNullElse(
postIds,
postRepository.findAll().stream().map(Post::getPostId).collect(Collectors.toList())));
}
public List<Image> findByPostIdAndIsNotCompleted(@NonNull String postId) {
return imageRepository.findByPostIdAndIsNotCompleted(postId);
}
public long countErrorImages() {
return imageRepository.countError();
}
public List<Image> findImagesByPostId(String postId) {
return imageRepository.findByPostId(postId);
}
public List<Post> findAllPosts() {
return postRepository.findAll();
}
public Optional<Post> findPostByPostId(String postId) {
return postRepository.findByPostId(postId);
}
public void stopImagesByPostIdAndIsNotCompleted(String postId) {
imageRepository.stopByPostIdAndIsNotCompleted(postId);
}
public Optional<Queued> findQueuedByThreadId(String threadId) {
return queuedRepository.findByThreadId(threadId);
}
public List<Queued> findAllQueued() {
return queuedRepository.findAll();
}
public Optional<Post> findById(Long aLong) {
return postRepository.findById(aLong);
}
public Optional<Image> findImageById(Long aLong) {
return imageRepository.findById(aLong);
}
public Optional<Queued> findQueuedById(Long aLong) {
return queuedRepository.findById(aLong);
}
public synchronized void setMetadata(Post post, Metadata metadata) {
if (metadataRepository.findByPostId(post.getPostId()).isEmpty()) {
metadata.setPostIdRef(post.getId());
metadataRepository.save(metadata);
}
}
public Optional<Metadata> findMetadataByPostId(String postId) {
return metadataRepository.findByPostId(postId);
}
public void updateImageStatus(Status status, Long id) {
imageRepository.updateStatus(status, id);
}
public void updateImageCurrent(long current, Long id) {
imageRepository.updateCurrent(current, id);
}
public void updateImageTotal(long total, Long id) {
imageRepository.updateTotal(total, id);
}
public Optional<LogEvent> findEventById(Long id) {
return eventRepository.findById(id);
}
public List<LogEvent> findAllEvents() {
return eventRepository.findAll();
}
public void clearQueueLinks() {
queuedRepository.deleteAll();
}
public synchronized void sortPostsByRank() {
List<Post> posts = findAllPosts();
posts.sort(Comparator.comparing(Post::getAddedOn));
for (int i = 0; i < posts.size(); i++) {
posts.get(i).setRank(i);
updatePostRank(i, posts.get(i).getId());
}
}
}
@@ -0,0 +1,40 @@
package me.vripper.services;
import java.util.concurrent.atomic.AtomicLong;
import lombok.Getter;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Service
@EnableScheduling
public class DownloadSpeedService {
private final AtomicLong read = new AtomicLong(0);
private final EventBus eventBus;
@Getter private long currentValue;
private boolean allowWrite = false;
public DownloadSpeedService(EventBus eventBus) {
this.eventBus = eventBus;
}
public void increase(long read) {
if (allowWrite) {
this.read.addAndGet(read);
}
}
@Scheduled(fixedDelay = 1000)
private void calc() {
allowWrite = false;
long newValue = read.getAndSet(0);
if (newValue != currentValue) {
currentValue = newValue;
eventBus.publishEvent(Event.wrap(Event.Kind.BYTES_PER_SECOND, currentValue));
}
allowWrite = true;
}
}
@@ -0,0 +1,42 @@
package me.vripper.services;
import lombok.Getter;
import me.vripper.download.DownloadService;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.services.domain.GlobalState;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Service
@EnableScheduling
public class GlobalStateService {
private final DownloadService downloadService;
private final DataService dataService;
private final EventBus eventBus;
@Getter private GlobalState currentState;
@Autowired
public GlobalStateService(
DownloadService downloadService, DataService dataService, EventBus eventBus) {
this.downloadService = downloadService;
this.dataService = dataService;
this.eventBus = eventBus;
}
@Scheduled(fixedDelay = 3000)
private void interval() {
GlobalState newGlobalState =
new GlobalState(
downloadService.runningCount(),
downloadService.pendingCount(),
dataService.countErrorImages());
if (!newGlobalState.equals(currentState)) {
currentState = newGlobalState;
eventBus.publishEvent(Event.wrap(Event.Kind.GLOBAL_STATE, currentState));
}
}
}
@@ -0,0 +1,112 @@
package me.vripper.services;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.HostException;
import me.vripper.exception.HtmlProcessorException;
import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@Service
@Slf4j
public class HostService {
private final ConnectionService cm;
private final HtmlProcessorService htmlProcessorService;
@Autowired
public HostService(ConnectionService cm, HtmlProcessorService htmlProcessorService) {
this.cm = cm;
this.htmlProcessorService = htmlProcessorService;
}
public Response getResponse(final String url, final HttpClientContext context)
throws HostException {
String basePage;
HttpClient client = cm.getClient().build();
HttpGet httpGet = cm.buildHttpGet(url, context);
Header[] headers;
log.debug(String.format("Requesting %s", url));
try (CloseableHttpResponse response =
(CloseableHttpResponse) client.execute(httpGet, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new HostException(
String.format(
"Unexpected response code: %d", response.getStatusLine().getStatusCode()));
}
headers = response.getAllHeaders();
basePage = EntityUtils.toString(response.getEntity());
log.debug(String.format("%s response: %n%s", url, basePage));
EntityUtils.consumeQuietly(response.getEntity());
} catch (IOException e) {
throw new HostException(e);
}
try {
log.debug(String.format("Cleaning %s response", url));
return new Response(htmlProcessorService.clean(basePage), headers);
} catch (HtmlProcessorException e) {
throw new HostException(e);
}
}
public String appendUri(String uri, String appendQuery) throws URISyntaxException {
URI oldUri = new URI(uri);
String newQuery = oldUri.getQuery();
if (newQuery == null) {
newQuery = appendQuery;
} else {
newQuery += "&" + appendQuery;
}
return new URI(
oldUri.getScheme(),
oldUri.getAuthority(),
oldUri.getPath(),
newQuery,
oldUri.getFragment())
.toString();
}
public String getDefaultImageName(final String imgUrl) {
String imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1);
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle));
return imgUrl;
}
@Getter
public static class Response {
private final Document document;
private final Header[] headers;
protected Response(Document document, Header[] headers) {
this.document = document;
this.headers = headers;
}
}
@Getter
public static class NameUrl {
private final String name;
private final String url;
public NameUrl(String name, String url) {
this.name = name;
this.url = url;
}
}
}
@@ -0,0 +1,22 @@
package me.vripper.services;
import me.vripper.exception.HtmlProcessorException;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@Service
public class HtmlProcessorService {
public Document clean(String htmlContent) throws HtmlProcessorException {
try {
TagNode clean = new HtmlCleaner().clean(htmlContent);
return new DomSerializer(new CleanerProperties()).createDOM(clean);
} catch (Exception e) {
throw new HtmlProcessorException(e);
}
}
}
@@ -0,0 +1,49 @@
package me.vripper.services;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import me.vripper.jpa.domain.Post;
import me.vripper.tasks.MetadataRunnable;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class MetadataService {
private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
private final ThreadPoolService threadPoolService;
public MetadataService(ThreadPoolService threadPoolService) {
this.threadPoolService = threadPoolService;
}
public void startFetchingMetadata(Post post) {
MetadataRunnable runnable = new MetadataRunnable(post);
threadPoolService.getGeneralExecutor().submit(runnable);
fetchingMetadata.put(post.getPostId(), runnable);
}
public void stopFetchingMetadata(List<String> postIds) {
List<MetadataRunnable> stopping = new ArrayList<>();
for (Map.Entry<String, MetadataRunnable> entry : this.fetchingMetadata.entrySet()) {
if (postIds.contains(entry.getKey())) {
stopping.add(entry.getValue());
entry.getValue().stop();
}
}
while (!stopping.isEmpty()) {
stopping.removeIf(MetadataRunnable::isFinished);
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
postIds.forEach(fetchingMetadata::remove);
}
}
@@ -0,0 +1,125 @@
package me.vripper.services;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.concurrent.locks.ReentrantLock;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import me.vripper.exception.RenameException;
import me.vripper.jpa.domain.Post;
import me.vripper.services.domain.Settings;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class PathService {
private final DataService dataService;
@Getter private final ReentrantLock directoryAccess = new ReentrantLock();
public PathService(DataService dataService) {
this.dataService = dataService;
}
public final File calcDownloadDirectory(Post post, Settings settings) {
return new File(settings.getDownloadPath(), post.getDownloadDirectory());
}
private File getRootFolder(
@NonNull String forum, @NonNull String threadTitle, Settings settings) {
File sourceFolder =
settings.getSubLocation()
? new File(settings.getDownloadPath(), sanitize(forum))
: new File(settings.getDownloadPath());
return settings.getThreadSubLocation() ? new File(sourceFolder, threadTitle) : sourceFolder;
}
public final void createDefaultPostFolder(Post post, Settings settings) {
File downloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
downloadDirectory =
new File(
downloadDirectory,
settings.getAppendPostId()
? sanitize(post.getTitle()) + "_" + post.getPostId()
: sanitize(post.getTitle()));
downloadDirectory = makeDir(downloadDirectory);
post.setDownloadDirectory(
downloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
}
public final void rename(@NonNull String postId, @NonNull String altName, Settings settings)
throws RenameException {
Post post = dataService.findPostByPostId(postId).orElseThrow();
if (altName.equals(post.getTitle())) {
return;
}
// Download have not started yet
if (post.getDownloadDirectory() == null) {
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
return;
}
File newDownloadDirectory = getRootFolder(post.getForum(), post.getThreadTitle(), settings);
newDownloadDirectory = new File(newDownloadDirectory, sanitize(altName));
File currentDownloadDirectory = calcDownloadDirectory(post, settings);
try {
directoryAccess.lock();
Files.move(
currentDownloadDirectory.toPath(),
newDownloadDirectory.toPath(),
StandardCopyOption.ATOMIC_MOVE);
post.setDownloadDirectory(
newDownloadDirectory.getAbsolutePath().replace(settings.getDownloadPath(), ""));
dataService.updatePostDownloadDirectory(post.getDownloadDirectory(), post.getId());
post.setTitle(altName);
dataService.updatePostTitle(post.getTitle(), post.getId());
} catch (IOException e) {
throw new RenameException(
String.format(
"Failed to move files from %s to %s", currentDownloadDirectory, newDownloadDirectory),
e);
} finally {
directoryAccess.unlock();
}
}
private File makeDir(@NonNull final File sourceFolder) {
int counter = 1;
File folder = sourceFolder;
while (folder.exists()) {
folder = new File(sourceFolder.toString() + '.' + counter++);
}
if (!folder.mkdirs()) {
throw new RuntimeException(String.format("Failed to create the folder %s", sourceFolder));
}
return folder;
}
private String sanitize(final String folderName) {
String sanitizedFolderName =
folderName.replaceAll("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}", "_");
log.debug(String.format("%s sanitized to %s", folderName, sanitizedFolderName));
return sanitizedFolderName;
}
/**
* Will sanitize the image name and remove extension
*
* @param imageName path string
* @return Sanitized local path string
*/
public final String formatImageFileName(String imageName) {
return sanitize(imageName);
}
}
@@ -0,0 +1,72 @@
package me.vripper.services;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import jakarta.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.domain.MultiPostScanParser;
import me.vripper.services.domain.MultiPostScanResult;
import me.vripper.tasks.AddPostRunnable;
import me.vripper.tasks.AddQueuedRunnable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@Slf4j
public class PostService {
private final DataService dataService;
private final ThreadPoolService threadPoolService;
private final LoadingCache<Queued, MultiPostScanResult> cache;
private final Disposable disposable;
@Autowired
public PostService(
DataService dataService, ThreadPoolService threadPoolService, EventBus eventBus) {
this.dataService = dataService;
this.threadPoolService = threadPoolService;
cache =
Caffeine.newBuilder()
.expireAfterWrite(5, TimeUnit.MINUTES)
.build(multiPostItem -> new MultiPostScanParser(multiPostItem).parse());
disposable =
eventBus
.flux()
.filter(e -> e.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> this.cache.invalidateAll());
}
@PreDestroy
private void destroy() {
if (disposable != null) {
disposable.dispose();
}
}
public void processMultiPost(List<Queued> queuedList) {
for (Queued queued : queuedList) {
if (queued.getPostId() != null) {
threadPoolService
.getGeneralExecutor()
.submit(new AddPostRunnable(queued.getPostId(), queued.getThreadId()));
} else {
threadPoolService.getGeneralExecutor().submit(new AddQueuedRunnable(queued));
}
}
}
public MultiPostScanResult get(Queued queued) throws ExecutionException {
return cache.get(queued);
}
public void remove(String threadId) {
dataService.removeQueueLink(threadId);
}
}
@@ -0,0 +1,304 @@
package me.vripper.services;
import static java.nio.file.StandardOpenOption.*;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.exception.ValidationException;
import me.vripper.services.domain.Settings;
import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
@Service
@Setter
@Slf4j
public class SettingsService {
private final Path configPath;
private final Path customProxiesPath;
private final ObjectMapper om = new ObjectMapper();
private final Set<String> proxies = new HashSet<>();
private final EventBus eventBus;
@Getter private Settings settings = new Settings();
@Value("classpath:proxies.json")
private Resource defaultProxies;
public SettingsService(
@Value("${base.dir}") String baseDir,
@Value("${base.dir.name}") String baseDirName,
EventBus eventBus) {
this.eventBus = eventBus;
this.configPath = Paths.get(baseDir, baseDirName, "config.json");
this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
@PostConstruct
private void init() {
loadViperProxies();
restore();
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
private void loadViperProxies() {
try (InputStream defaultProxiesIs = defaultProxies.getInputStream()) {
List<String> defaultProxies = om.readValue(defaultProxiesIs, new TypeReference<>() {});
List<String> customProxies = new ArrayList<>();
if (customProxiesPath.toFile().exists() && customProxiesPath.toFile().isFile()) {
customProxies = om.readValue(customProxiesPath.toFile(), new TypeReference<>() {});
} else {
if (!customProxiesPath.toFile().createNewFile()) {
log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath());
} else {
try (FileWriter fw = new FileWriter(customProxiesPath.toFile())) {
fw.append("[]").append(System.lineSeparator());
fw.flush();
} catch (IOException e) {
log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath(), e);
}
}
}
proxies.addAll(customProxies);
proxies.addAll(defaultProxies);
} catch (IOException e) {
log.error("Failed to load vipergirls proxies list", e);
}
proxies.add("https://vipergirls.to");
}
public List<String> getProxies() {
return new ArrayList<>(proxies);
}
public void newSettings(Settings settings) {
if (settings.getVLogin() != null && settings.getVLogin()) {
if (!this.settings.getVPassword().equals(settings.getVPassword())) {
settings.setVPassword(DigestUtils.md5Hex(settings.getVPassword()));
}
} else {
settings.setVUsername("");
settings.setVPassword("");
settings.setVThanks(false);
settings.setVLogin(false);
}
this.settings = settings;
save();
eventBus.publishEvent(Event.wrap(Event.Kind.SETTINGS_UPDATE, settings));
}
public void restore() {
try {
if (configPath.toFile().exists()) {
settings = om.readValue(configPath.toFile(), Settings.class);
}
} catch (IOException e) {
log.error("Failed restore user settings", e);
settings = new Settings();
}
if (settings.getDownloadPath() == null) {
settings.setDownloadPath(System.getProperty("user.home"));
}
if (settings.getMaxThreads() == null) {
settings.setMaxThreads(4);
}
if (settings.getMaxTotalThreads() == null) {
settings.setMaxTotalThreads(0);
}
if (settings.getAutoStart() == null) {
settings.setAutoStart(true);
}
if (settings.getVLogin() == null) {
settings.setVLogin(false);
}
if (settings.getVUsername() == null) {
settings.setVUsername("");
}
if (settings.getVPassword() == null) {
settings.setVPassword("");
}
if (settings.getVThanks() == null) {
settings.setVThanks(false);
}
if (settings.getDesktopClipboard() == null) {
settings.setDesktopClipboard(false);
}
if (settings.getForceOrder() == null) {
settings.setForceOrder(false);
}
if (settings.getSubLocation() == null) {
settings.setSubLocation(false);
}
if (settings.getThreadSubLocation() == null) {
settings.setThreadSubLocation(false);
}
if (settings.getClearCompleted() == null) {
settings.setClearCompleted(false);
}
if (settings.getDarkTheme() == null) {
settings.setDarkTheme(false);
}
if (settings.getAppendPostId() == null) {
settings.setAppendPostId(true);
}
if (settings.getLeaveThanksOnStart() == null) {
settings.setLeaveThanksOnStart(false);
}
if (settings.getConnectionTimeout() == null) {
settings.setConnectionTimeout(30);
}
if (settings.getMaxAttempts() == null) {
settings.setMaxAttempts(5);
}
if (settings.getVProxy() == null || !proxies.contains(settings.getVProxy())) {
settings.setVProxy("https://vipergirls.to");
}
if (settings.getMaxEventLog() == null) {
settings.setMaxEventLog(1000);
}
try {
check(this.settings);
} catch (ValidationException e) {
log.error(
String.format("Your settings are invalid, either remove %s, or fix it", configPath), e);
SpringContext.close();
}
save();
}
public void save() {
try {
Files.write(
configPath, om.writeValueAsBytes(settings), CREATE, WRITE, TRUNCATE_EXISTING, SYNC);
} catch (IOException e) {
log.error("Failed to store user settings", e);
}
}
@PreDestroy
private void destroy() {
save();
}
public void check(Settings settings) throws ValidationException {
Path path;
try {
path = Paths.get(settings.getDownloadPath());
} catch (InvalidPathException e) {
throw new ValidationException(String.format("%s is invalid", settings.getDownloadPath()));
}
if (!Files.exists(path)) {
throw new ValidationException(String.format("%s does not exist", settings.getDownloadPath()));
} else if (!Files.isDirectory(path)) {
throw new ValidationException(
String.format("%s is not a directory", settings.getDownloadPath()));
}
if (settings.getMaxTotalThreads() < 0 || settings.getMaxTotalThreads() > 12) {
throw new ValidationException(
String.format(
"Invalid max global concurrent download settings, values must be in [%d,%d]", 0, 12));
}
if (settings.getMaxThreads() < 1 || settings.getMaxThreads() > 4) {
throw new ValidationException(
String.format(
"Invalid max concurrent download settings, values must be in [%d,%d]", 1, 4));
}
if (settings.getConnectionTimeout() < 1 || settings.getConnectionTimeout() > 300) {
throw new ValidationException(
String.format("Invalid connection timeout settings, values must be in [%d,%d]", 1, 300));
}
if (settings.getMaxAttempts() < 1 || settings.getMaxAttempts() > 10) {
throw new ValidationException(
String.format("Invalid maximum attempts settings, values must be in [%d,%d]", 1, 10));
}
if (settings.getMaxEventLog() < 100 || settings.getMaxEventLog() > 10_000) {
throw new ValidationException(
String.format(
"Invalid maximum event log record settings, values must be in [%d,%d]", 100, 10_000));
}
if (!settings.getVThanks()
&& settings.getLeaveThanksOnStart() != null
&& settings.getLeaveThanksOnStart()) {
throw new ValidationException("Invalid value, leave thanks must be checked");
}
}
public Theme getTheme() {
return new Theme(this.settings.getDarkTheme());
}
public void setTheme(Theme theme) {
this.settings.setDarkTheme(theme.darkTheme);
save();
}
@Getter
@NoArgsConstructor
public static class Theme {
@JsonProperty("darkTheme")
private boolean darkTheme;
Theme(boolean darkTheme) {
this.darkTheme = darkTheme;
}
}
}
@@ -0,0 +1,19 @@
package me.vripper.services;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import org.springframework.stereotype.Service;
@Service
public class ThreadPoolService {
@Getter
private final ExecutorService generalExecutor =
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
public void destroy() throws Exception {
generalExecutor.shutdown();
generalExecutor.awaitTermination(5, TimeUnit.SECONDS);
}
}
@@ -0,0 +1,158 @@
package me.vripper.services;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import me.vripper.event.Event;
import me.vripper.event.EventBus;
import me.vripper.exception.VripperException;
import me.vripper.jpa.domain.Post;
import me.vripper.tasks.LeaveThanksRunnable;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.Disposable;
@Service
@Slf4j
public class VGAuthService {
private final ConnectionService cm;
private final SettingsService settingsService;
private final ThreadPoolService threadPoolService;
private final Disposable disposable;
@Getter private final HttpClientContext context = HttpClientContext.create();
@Getter private boolean authenticated = false;
@Getter private String loggedUser = "";
private final EventBus eventBus;
@Autowired
public VGAuthService(
ConnectionService cm,
SettingsService settingsService,
ThreadPoolService threadPoolService,
EventBus eventBus) {
this.cm = cm;
this.settingsService = settingsService;
this.threadPoolService = threadPoolService;
this.eventBus = eventBus;
disposable =
eventBus
.flux()
.filter(p -> p.getKind().equals(Event.Kind.SETTINGS_UPDATE))
.subscribe(e -> authenticate());
}
@PostConstruct
private void init() {
context.setCookieStore(new BasicCookieStore());
authenticate();
}
@PreDestroy
private void destroy() {
disposable.dispose();
}
public void authenticate() {
log.info("Authenticating using ViperGirls credentials");
authenticated = false;
if (!settingsService.getSettings().getVLogin()) {
log.debug("Authentication option is disabled");
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
return;
}
String username = settingsService.getSettings().getVUsername();
String password = settingsService.getSettings().getVPassword();
if (username == null || password == null || username.isEmpty() || password.isEmpty()) {
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty");
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
return;
}
HttpPost postAuth =
cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/login.php?do=login", null);
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("vb_login_username", username));
params.add(new BasicNameValuePair("cookieuser", "1"));
params.add(new BasicNameValuePair("do", "login"));
params.add(new BasicNameValuePair("vb_login_md5password", password));
try {
postAuth.setEntity(new UrlEncodedFormEntity(params));
} catch (Exception e) {
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
return;
}
postAuth.addHeader("Referer", settingsService.getSettings().getVProxy());
postAuth.addHeader(
"Host",
settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", ""));
CloseableHttpClient client = cm.getClient().build();
try (CloseableHttpResponse response = client.execute(postAuth, context)) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new VripperException(
String.format(
"Unexpected response code returned %s", response.getStatusLine().getStatusCode()));
}
String responseBody = EntityUtils.toString(response.getEntity());
log.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
EntityUtils.consumeQuietly(response.getEntity());
if (context.getCookieStore().getCookies().stream()
.map(Cookie::getName)
.noneMatch(e -> e.equals("vg_userid"))) {
log.error(
String.format(
"Failed to authenticate user with %s, missing vg_userid cookie",
settingsService.getSettings().getVProxy()));
return;
}
} catch (Exception e) {
context.getCookieStore().clear();
loggedUser = "";
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
return;
}
authenticated = true;
loggedUser = username;
log.info(String.format("Authenticated: %s", username));
eventBus.publishEvent(Event.wrap(Event.Kind.VG_USER, loggedUser));
}
public void leaveThanks(Post post) {
threadPoolService
.getGeneralExecutor()
.submit(new LeaveThanksRunnable(post, authenticated, context));
}
}
@@ -0,0 +1,31 @@
package me.vripper.services;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import me.vripper.exception.XpathException;
import org.springframework.stereotype.Service;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@Service
public class XpathService {
private final XPath xpath = XPathFactory.newInstance().newXPath();
public Node getAsNode(Node source, String xpathExpression) throws XpathException {
try {
return (Node) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE);
} catch (Exception e) {
throw new XpathException(e);
}
}
public NodeList getAsNodeList(Node source, String xpathExpression) throws XpathException {
try {
return (NodeList) xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET);
} catch (Exception e) {
throw new XpathException(e);
}
}
}
@@ -0,0 +1,39 @@
package me.vripper.services.domain;
import java.util.Objects;
import lombok.Getter;
@Getter
public class DownloadSpeed {
private final String speed;
public DownloadSpeed(long bytes) {
speed = formatSI(bytes);
}
private String formatSI(long bytes) {
return humanReadableByteCount(bytes, false);
}
private String humanReadableByteCount(long bytes, boolean si) {
int unit = si ? 1000 : 1024;
if (bytes < unit) return bytes + " B";
int exp = (int) (Math.log(bytes) / Math.log(unit));
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DownloadSpeed that = (DownloadSpeed) o;
return Objects.equals(speed, that.speed);
}
@Override
public int hashCode() {
return Objects.hash(speed);
}
}
@@ -0,0 +1,31 @@
package me.vripper.services.domain;
import java.util.Objects;
import lombok.Getter;
@Getter
public class GlobalState {
private final long running;
private final long remaining;
private final long error;
public GlobalState(long running, long remaining, long error) {
this.running = running;
this.remaining = remaining;
this.error = error;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GlobalState that = (GlobalState) o;
return running == that.running && remaining == that.remaining && error == that.error;
}
@Override
public int hashCode() {
return Objects.hash(running, remaining, error);
}
}
@@ -0,0 +1,35 @@
package me.vripper.services.domain;
import java.util.List;
import lombok.Getter;
@Getter
public class MultiPostItem {
private final String threadId;
private final String postId;
private final int number;
private final String title;
private final int imageCount;
private final String url;
private final List<String> previews;
private final String hosts;
public MultiPostItem(
String threadId,
String postId,
int number,
String title,
int imageCount,
String url,
List<String> previews,
String hosts) {
this.threadId = threadId;
this.postId = postId;
this.number = number;
this.title = title;
this.imageCount = imageCount;
this.previews = previews;
this.url = url;
this.hosts = hosts;
}
}
@@ -0,0 +1,113 @@
package me.vripper.services.domain;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import me.vripper.SpringContext;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.SettingsService;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
public class MultiPostScanHandler extends DefaultHandler {
private final Queued queued;
private final Collection<Host> supportedHosts;
private final SettingsService settingsService;
private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
private final List<MultiPostItem> posts = new ArrayList<>();
private String error;
private List<String> previews = new ArrayList<>();
private String threadTitle;
private String postId;
private String postTitle;
private int imageCount;
private int postCounter;
private int previewCounter = 0;
public MultiPostScanHandler(Queued queued) {
this.queued = queued;
supportedHosts = SpringContext.getBeansOfType(Host.class).values();
settingsService = SpringContext.getBean(SettingsService.class);
}
public MultiPostScanResult getScanResult() {
return new MultiPostScanResult(posts, error);
}
@Override
public void startDocument() {}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "error":
error = attributes.getValue("details");
break;
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
imageCount = Integer.parseInt(attributes.getValue("imagecount").trim());
postId = attributes.getValue("id").trim();
postCounter = Integer.parseInt(attributes.getValue("number").trim());
postTitle =
Optional.ofNullable(attributes.getValue("title"))
.map(e -> e.trim().isEmpty() ? null : e.trim())
.orElse(threadTitle);
break;
case "image":
Optional.ofNullable(attributes.getValue("main_url"))
.map(String::trim)
.flatMap(
mainUrl ->
supportedHosts.stream().filter(host -> host.isSupported(mainUrl)).findFirst())
.ifPresent(
host ->
Optional.ofNullable(hostMap.get(host))
.ifPresentOrElse(
AtomicInteger::incrementAndGet,
() -> hostMap.put(host, new AtomicInteger(1))));
if (previewCounter++ < 4) {
Optional.ofNullable(attributes.getValue("thumb_url"))
.map(String::trim)
.ifPresent(previews::add);
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equalsIgnoreCase(qName)) {
if (imageCount != 0) {
posts.add(
new MultiPostItem(
queued.getThreadId(),
postId,
postCounter,
postTitle,
imageCount,
String.format(
"%s/threads/?p=%s&viewfull=1#post%s",
settingsService.getSettings().getVProxy(), postId, postId),
previews,
hostMap.entrySet().stream()
.filter(v -> v.getValue().get() > 0)
.map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")")
.collect(Collectors.joining(", "))));
queued.increment();
}
previewCounter = 0;
previews = new ArrayList<>();
hostMap.clear();
}
}
@Override
public void endDocument() {
queued.done();
}
}
@@ -0,0 +1,90 @@
package me.vripper.services.domain;
import java.io.BufferedInputStream;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.SAXParserFactory;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.jpa.domain.Queued;
import me.vripper.services.ConnectionService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
@Slf4j
public class MultiPostScanParser {
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final Queued queued;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
private final SettingsService settingsService;
public MultiPostScanParser(Queued queued) {
this.queued = queued;
cm = SpringContext.getBean(ConnectionService.class);
VGAuthService = SpringContext.getBean(VGAuthService.class);
settingsService = SpringContext.getBean(SettingsService.class);
}
public MultiPostScanResult parse() throws PostParseException {
log.debug(String.format("Parsing thread %s", queued));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php");
uriBuilder.setParameter("t", queued.getThreadId());
httpGet = cm.buildHttpGet(uriBuilder.build(), null);
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
MultiPostScanHandler multiPostScanHandler = new MultiPostScanHandler(queued);
AtomicReference<Throwable> thr = new AtomicReference<>();
log.debug(String.format("Requesting %s", httpGet));
MultiPostScanResult multiPostScanResult =
Failsafe.with(cm.getRetryPolicy())
.onFailure(e -> thr.set(e.getFailure()))
.get(
() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response =
(CloseableHttpResponse)
connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(
String.format(
"Unexpected response code '%d' for %s",
response.getStatusLine().getStatusCode(), httpGet));
}
try {
factory
.newSAXParser()
.parse(
new BufferedInputStream(response.getEntity().getContent()),
multiPostScanHandler);
return multiPostScanHandler.getScanResult();
} catch (Exception e) {
throw new PostParseException(
String.format("Failed to parse thread %s", queued), e);
} finally {
EntityUtils.consumeQuietly(response.getEntity());
}
}
});
if (thr.get() != null) {
log.error(String.format("parsing failed for thread %s", queued), thr.get());
throw new PostParseException(thr.get());
}
return multiPostScanResult;
}
}
@@ -0,0 +1,15 @@
package me.vripper.services.domain;
import java.util.List;
import lombok.Getter;
@Getter
public class MultiPostScanResult {
private final List<MultiPostItem> posts;
private final String error;
public MultiPostScanResult(List<MultiPostItem> posts, String error) {
this.posts = posts;
this.error = error;
}
}
@@ -0,0 +1,114 @@
package me.vripper.services.domain;
import java.util.Collection;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.host.Host;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
import me.vripper.services.SettingsService;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
@Slf4j
class PostScanHandler extends DefaultHandler {
private final Collection<Host> supportedHosts;
private final String threadId;
private final String postId;
private final String postUrl;
private final Set<Image> images = new HashSet<>();
private Set<String> previews = new HashSet<>();
private String threadTitle;
private String forum;
private String userHash;
private int index = 0;
private Post parsedPost;
PostScanHandler(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
supportedHosts = SpringContext.getBeansOfType(Host.class).values();
SettingsService settingsService = SpringContext.getBean(SettingsService.class);
postUrl =
String.format(
"%s/threads/%s/?p=%s&viewfull=1#post%s",
settingsService.getSettings().getVProxy(), this.threadId, this.postId, this.postId);
}
public PostScanResult getParsedPost() {
return new PostScanResult(parsedPost, images);
}
@Override
public void startDocument() {}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) {
switch (qName.toLowerCase()) {
case "forum":
forum = attributes.getValue("title").trim();
break;
case "user":
userHash = attributes.getValue("hash").trim();
break;
case "thread":
threadTitle = attributes.getValue("title").trim();
break;
case "post":
String postTitle =
Optional.ofNullable(attributes.getValue("title"))
.map(e -> e.trim().isEmpty() ? null : e.trim())
.orElse(threadTitle);
parsedPost = new Post(postTitle, postUrl, postId, threadId, threadTitle, forum, userHash);
break;
case "image":
index++;
String thumbUrl = attributes.getValue("thumb_url").trim();
if (previews.size() < 4) {
previews.add(thumbUrl);
}
String mainUrl =
Optional.ofNullable(attributes.getValue("main_url")).map(String::trim).orElse(null);
if (mainUrl != null) {
Host foundHost =
supportedHosts.stream()
.filter(host -> host.isSupported(mainUrl))
.findFirst()
.orElse(null);
if (foundHost != null) {
log.debug(
String.format(
"Found supported host %s for %s",
foundHost.getClass().getSimpleName(), mainUrl));
images.add(new Image(postId, mainUrl, thumbUrl, foundHost, index));
} else {
log.warn(String.format("unsupported host for %s, skipping", mainUrl));
}
}
break;
}
}
@Override
public void endElement(String uri, String localName, String qName) {
if ("post".equalsIgnoreCase(qName)) {
parsedPost.setTotal(images.size());
if (!previews.isEmpty()) {
parsedPost.setPreviews(previews);
}
parsedPost.setHosts(
images.stream().map(Image::getHost).map(Host::getHost).collect(Collectors.toSet()));
index = 0;
previews = new HashSet<>();
}
}
}
@@ -0,0 +1,85 @@
package me.vripper.services.domain;
import java.net.URISyntaxException;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.SAXParserFactory;
import lombok.extern.slf4j.Slf4j;
import me.vripper.SpringContext;
import me.vripper.exception.DownloadException;
import me.vripper.exception.PostParseException;
import me.vripper.services.ConnectionService;
import me.vripper.services.SettingsService;
import me.vripper.services.VGAuthService;
import net.jodah.failsafe.Failsafe;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.util.EntityUtils;
@Slf4j
public class PostScanParser {
private static final SAXParserFactory factory = SAXParserFactory.newInstance();
private final String threadId;
private final String postId;
private final ConnectionService cm;
private final VGAuthService VGAuthService;
private final SettingsService settingsService;
public PostScanParser(String threadId, String postId) {
this.threadId = threadId;
this.postId = postId;
cm = SpringContext.getBean(ConnectionService.class);
VGAuthService = SpringContext.getBean(VGAuthService.class);
settingsService = SpringContext.getBean(SettingsService.class);
}
public PostScanResult parse() throws PostParseException {
log.debug(String.format("Parsing post %s", postId));
HttpGet httpGet;
try {
URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php");
uriBuilder.setParameter("p", postId);
httpGet = cm.buildHttpGet(uriBuilder.build(), null);
} catch (URISyntaxException e) {
throw new PostParseException(e);
}
AtomicReference<Throwable> thr = new AtomicReference<>();
PostScanHandler postScanHandler = new PostScanHandler(threadId, postId);
log.debug(String.format("Requesting %s", httpGet));
PostScanResult post = getPost(httpGet, postScanHandler, thr);
if (thr.get() != null) {
log.error(
String.format("parsing failed for thread %s, post %s", threadId, postId), thr.get());
throw new PostParseException(thr.get());
}
return post;
}
private PostScanResult getPost(
HttpGet httpGet, PostScanHandler postScanHandler, AtomicReference<Throwable> thr) {
return Failsafe.with(cm.getRetryPolicy())
.onFailure(e -> thr.set(e.getFailure()))
.get(
() -> {
HttpClient connection = cm.getClient().build();
try (CloseableHttpResponse response =
(CloseableHttpResponse) connection.execute(httpGet, VGAuthService.getContext())) {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
throw new DownloadException(
String.format(
"Unexpected response code '%d' for %s",
response.getStatusLine().getStatusCode(), httpGet));
}
factory.newSAXParser().parse(response.getEntity().getContent(), postScanHandler);
EntityUtils.consumeQuietly(response.getEntity());
return postScanHandler.getParsedPost();
}
});
}
}
@@ -0,0 +1,23 @@
package me.vripper.services.domain;
import java.util.Optional;
import java.util.Set;
import lombok.Getter;
import me.vripper.jpa.domain.Image;
import me.vripper.jpa.domain.Post;
public class PostScanResult {
private final Post post;
@Getter private final Set<Image> images;
public PostScanResult(Post post, Set<Image> images) {
this.post = post;
this.images = images;
}
public Optional<Post> getPost() {
return Optional.ofNullable(post);
}
}

Some files were not shown because too many files have changed in this diff Show More