mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbe5800c08 | ||
|
|
b200226060 | ||
|
|
0d22b054c9 | ||
|
|
9d18dc8897 | ||
|
|
f42d412b86 | ||
|
|
f8370801c2 | ||
|
|
51dcaa1946 | ||
|
|
a833a0b1b2 | ||
|
|
79739b80f9 | ||
|
|
26cf60b741 | ||
|
|
bc45deed2a | ||
|
|
b1e3353a1d | ||
|
|
c018033d2f | ||
|
|
b0f5a15943 | ||
|
|
8db4d406ad | ||
|
|
47f1705abb | ||
|
|
816fb84549 | ||
|
|
5213ed7129 | ||
|
|
2483c5fa00 | ||
|
|
dfc5810fd3 | ||
|
|
fad418134e | ||
|
|
9dd91bfc07 | ||
|
|
5bd42e2ce5 | ||
|
|
e18832b154 | ||
|
|
41feb53a83 | ||
|
|
6f166b42c4 | ||
|
|
e8ee579a37 | ||
|
|
41ce99995b | ||
|
|
5caeef0e70 | ||
|
|
96ef0cb234 | ||
|
|
641b3e3e7b | ||
|
|
08de4dc5b0 | ||
|
|
57680a6053 | ||
|
|
53fd14935b | ||
|
|
90af03d68f | ||
|
|
265614585b | ||
|
|
4503d540ed | ||
|
|
b08b730dff | ||
|
|
06d636329e | ||
|
|
85313600fc | ||
|
|
7d451397f9 | ||
|
|
1c94b93597 | ||
|
|
8f09596453 | ||
|
|
6ce05e78b3 | ||
|
|
67a375150c | ||
|
|
fba5f670a0 | ||
|
|
7d083630cc | ||
|
|
335cbf70b4 | ||
|
|
e25d306f1d | ||
|
|
f395338a9d | ||
|
|
58a40682cc | ||
|
|
ccc4032bb0 | ||
|
|
1c5b8a0397 | ||
|
|
cc2ce9482d | ||
|
|
41f1775360 | ||
|
|
733f1a08ac | ||
|
|
0cb0ee0100 | ||
|
|
cb88e39c83 | ||
|
|
6375c0aed6 | ||
|
|
40ca4cf86b | ||
|
|
b3a798388b | ||
|
|
6e69b2bce8 | ||
|
|
bb6dcd77b9 | ||
|
|
42e1aa73e2 | ||
|
|
26fde22af2 | ||
|
|
798f39d54f | ||
|
|
3a21dfac38 | ||
|
|
e8355b88b9 | ||
|
|
379660236d | ||
|
|
c86ee91c27 | ||
|
|
ff9d253242 | ||
|
|
53b45e00ba | ||
|
|
247b12a525 | ||
|
|
7a8d332e6e | ||
|
|
749a6f18d2 | ||
|
|
4107048f12 | ||
|
|
f5d9ab85a2 | ||
|
|
b82ee7b494 | ||
|
|
8a31dfc012 | ||
|
|
734c1b72c7 | ||
|
|
7abb7df1f6 | ||
|
|
0a7fa99a64 | ||
|
|
691026c8a9 | ||
|
|
3ef004b947 | ||
|
|
1b22a93e28 | ||
|
|
2639df0aaa | ||
|
|
d5ae5d57d1 | ||
|
|
537d76d7be | ||
|
|
c1247faacc | ||
|
|
efcc685f30 | ||
|
|
c825b22ce4 | ||
|
|
6ef5d5474f | ||
|
|
bd25801c6c | ||
|
|
377b93c1ac | ||
|
|
d498d20f6b | ||
|
|
d18544d414 | ||
|
|
3c872b79bd | ||
|
|
37900cac49 | ||
|
|
be57c76d93 | ||
|
|
8b52ad7858 | ||
|
|
a0976bde02 | ||
|
|
15c97d5cfc | ||
|
|
905cbd5a35 | ||
|
|
5c0e2e6185 | ||
|
|
4836d2dacb | ||
|
|
bc81ec1ab5 | ||
|
|
8a4d039a37 | ||
|
|
468639264a | ||
|
|
9b649b470f | ||
|
|
edd5cf2bf7 | ||
|
|
67c605e640 | ||
|
|
8d4b878656 | ||
|
|
bf532dd1d0 | ||
|
|
eb1a125e59 | ||
|
|
256659a72d | ||
|
|
dbf3aa7e7d | ||
|
|
a75ce181ff | ||
|
|
37ae6b4ebf | ||
|
|
6631a6a95b | ||
|
|
918f8578ad | ||
|
|
10fa9f68ae | ||
|
|
6b5c9ccb0f | ||
|
|
652e8f1a9d | ||
|
|
fe53ae733d | ||
|
|
690dab2ae8 | ||
|
|
530542163e | ||
|
|
cf7f4c75c0 | ||
|
|
c6b395cf4c | ||
|
|
e72edeed62 | ||
|
|
3f22d53f89 | ||
|
|
08c0fa62b1 | ||
|
|
4c79cca079 | ||
|
|
67096242a2 | ||
|
|
f66112375d | ||
|
|
c5a082e248 | ||
|
|
c2a1b0aa33 | ||
|
|
a89dcc96d8 | ||
|
|
b2a10eafd9 | ||
|
|
f027aff89d | ||
|
|
b4eaf0d1c4 | ||
|
|
9d2975faaa | ||
|
|
0c0fdf9d78 | ||
|
|
e17afa0ef3 | ||
|
|
3d3eb6aa36 | ||
|
|
e22132255e | ||
|
|
ef041526ec | ||
|
|
704311dd9a | ||
|
|
9617a6f20f | ||
|
|
bc17442961 | ||
|
|
574d6818c1 | ||
|
|
5c416055ee | ||
|
|
fa30458dc6 | ||
|
|
638af93225 | ||
|
|
2c6ca741c7 | ||
|
|
d7f2df2540 | ||
|
|
b85ade1dba | ||
|
|
1c61a0b34e | ||
|
|
46b9449969 | ||
|
|
cc1bbd1231 | ||
|
|
b29cba2911 | ||
|
|
13bde3aa2c | ||
|
|
35b4f5fc97 | ||
|
|
1b0482b7d4 | ||
|
|
c9ff6eeab6 | ||
|
|
5ac57b743d | ||
|
|
f7b219701f | ||
|
|
d009aa3f8c | ||
|
|
078d49f883 | ||
|
|
f1bf517ba5 | ||
|
|
6269ed6f6e | ||
|
|
43b8dedbb1 | ||
|
|
978be7bd49 | ||
|
|
1b7f3aef62 | ||
|
|
7d24ee4867 | ||
|
|
269a3475b0 | ||
|
|
18e2bc6afc | ||
|
|
c6ef54e115 | ||
|
|
441b2831fc | ||
|
|
3cf4d60eca | ||
|
|
d228c89a1c | ||
|
|
f3b592e81b | ||
|
|
a5ff659fb4 | ||
|
|
ab80e2f3b3 | ||
|
|
2ef082f87e | ||
|
|
a1b97e76a7 | ||
|
|
185a078b4d | ||
|
|
28a4c944ee | ||
|
|
23d3553d99 | ||
|
|
fb09e4c46b | ||
|
|
4feb6f5fe1 | ||
|
|
df1b9efbbb | ||
|
|
553924fa1d |
@@ -0,0 +1,43 @@
|
||||
# This workflow will build a package using Maven and then publish it to GitHub packages when a release is created
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: Release
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
|
||||
settings-path: ${{ github.workspace }} # location for the settings.xml file
|
||||
|
||||
- name: Build with Maven
|
||||
run: mvn -B install --file pom.xml
|
||||
- name: Rename GUI artifact
|
||||
run: mv vripper-gui/target/vripper-gui-*.jar vripper-gui/target/vripper-gui.jar
|
||||
- name: Upload GUI Binary
|
||||
uses: skx/github-action-publish-binaries@master
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
args: 'vripper-gui/target/vripper-gui.jar'
|
||||
- name: Rename WEB artifact
|
||||
run: mv vripper-web/target/vripper-web-*.jar vripper-web/target/vripper-web.jar
|
||||
- name: Upload WEB Binary
|
||||
uses: skx/github-action-publish-binaries@master
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
args: 'vripper-web/target/vripper-web.jar'
|
||||
|
||||
+2
-1
@@ -3,4 +3,5 @@
|
||||
/**/*/target
|
||||
/**/*/build-dir
|
||||
/**/*/java-runtime
|
||||
.idea
|
||||
.idea
|
||||
.vripper
|
||||
+560
-4
@@ -1,119 +1,635 @@
|
||||
# Changelog
|
||||
|
||||
## [2.3.2] - 2019-11-23
|
||||
## [4.0.0] - 2023-05-03
|
||||
|
||||
- Drop electron app
|
||||
- Use javafx for the desktop app
|
||||
- Re-written in kotlin
|
||||
- Several enhancements
|
||||
|
||||
### Changed
|
||||
|
||||
## [3.5.4] - 2021-05-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix download queue bug
|
||||
|
||||
## [3.5.3] - 2021-05-24
|
||||
|
||||
### Changed
|
||||
|
||||
- Update donation infos
|
||||
|
||||
## [3.5.1] - 2021-05-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fix
|
||||
|
||||
## [3.5.0] - 2021-05-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Download queue bug fix
|
||||
- Save window size and position
|
||||
- Resize confirmation box
|
||||
- Enhance metadata interruption
|
||||
- Add Added on and Order columns
|
||||
|
||||
## [3.4.2] - 2021-05-10
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix potential issue with imagebam host
|
||||
- Enhance download queue logic
|
||||
- Code source enhancements
|
||||
- Fix app icons
|
||||
- Fix race condition bug
|
||||
|
||||
## [3.3.8] - 2021-05-05
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix imagebam host
|
||||
- Fix font path
|
||||
|
||||
## [3.3.6] - 2021-03-20
|
||||
|
||||
### Changed
|
||||
|
||||
- Optimize build size
|
||||
|
||||
## [3.3.5] - 2021-03-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fixe
|
||||
|
||||
## [3.3.4] - 2021-03-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [3.3.3] - 2021-03-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Remove unnecessary event logs
|
||||
|
||||
## [3.3.2] - 2021-03-11
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a bug with event logs
|
||||
- Change event names and add some icons
|
||||
|
||||
## [3.3.1] - 2021-02-18
|
||||
|
||||
### Added
|
||||
|
||||
- Add about menu
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix README.md
|
||||
|
||||
## [3.3.0] - 2021-02-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Add support for proxies
|
||||
- Add a Event Logs
|
||||
- Fix bugs with the database
|
||||
|
||||
## [3.2.3] - 2021-01-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Finished posts are not cleared from the UI
|
||||
|
||||
## [3.2.2] - 2021-01-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Set total download to max value
|
||||
|
||||
## [3.2.1] - 2021-01-12
|
||||
|
||||
### Added
|
||||
|
||||
- Add settings for connection timeout
|
||||
- Add settings for maximum attempts
|
||||
|
||||
### Changed
|
||||
|
||||
- Enhance posts rename logic
|
||||
- Dependencies update
|
||||
- Switch reactive from rx-java to reactor
|
||||
|
||||
## [3.2.0] - 2020-11-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix build issue
|
||||
|
||||
## [3.1.0] - 2020-11-16
|
||||
|
||||
### Changed
|
||||
|
||||
- UI revamp
|
||||
- Fix bugs
|
||||
|
||||
## [3.0.11] - 2020-10-31
|
||||
|
||||
### Added
|
||||
|
||||
- Support for vipr.im
|
||||
|
||||
## [3.0.10] - 2020-10-04
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bug on settings update
|
||||
|
||||
## [3.0.9] - 2020-09-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Revert to appending post id to download folders
|
||||
- Fix a bug that lead to adding duplicate posts
|
||||
- Remove old unused settings
|
||||
|
||||
## [3.0.8] - 2020-08-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fix which breaks concurrent downloads
|
||||
|
||||
## [3.0.7] - 2020-08-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix MacOs build
|
||||
|
||||
## [3.0.6] - 2020-08-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix TurboImageHost
|
||||
|
||||
## [3.0.5] - 2020-08-17
|
||||
|
||||
### Changed
|
||||
|
||||
- Update packages
|
||||
- Host class refactoring
|
||||
|
||||
## [3.0.4] - 2020-08-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Improve rename UI
|
||||
|
||||
## [3.0.3] - 2020-08-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bugs with electron app
|
||||
|
||||
## [3.0.2] - 2020-08-15
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.1] - 2020-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bugs with download queue
|
||||
|
||||
## [3.0.0] - 2020-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Major rewrites
|
||||
- Introducing a proper database for persistence
|
||||
|
||||
## [2.12.1] - 2020-05-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix alternative name bug
|
||||
|
||||
## [2.12.0] - 2020-05-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Use new user hash from the api
|
||||
|
||||
## [2.11.7] - 2020-05-08
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix bug with paths creation
|
||||
- Move Open Link button
|
||||
- Rename rename all title
|
||||
|
||||
## [2.11.6] - 2020-05-04
|
||||
|
||||
### Changed
|
||||
|
||||
- PNG filenames are now correctly named
|
||||
- Fix issue with special characters in post titles
|
||||
|
||||
## [2.11.5] - 2020-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Reduce rows buffer size
|
||||
|
||||
## [2.11.4] - 2020-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Code cleanup
|
||||
- Dependencies update
|
||||
|
||||
## [2.11.3] - 2020-04-30
|
||||
|
||||
### Changed
|
||||
|
||||
- Source code restructure
|
||||
- UI fixes and enhancements
|
||||
|
||||
### Added
|
||||
|
||||
- Add menu entry to rename a gallery to first alternative title
|
||||
- Add menu on toolbar for massive rename to first alternative title
|
||||
|
||||
## [2.11.2] - 2020-04-26
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [2.11.1] - 2020-04-25
|
||||
|
||||
### Changed
|
||||
|
||||
- Code source maintenance
|
||||
|
||||
### Added
|
||||
|
||||
- Added option to rename gallery
|
||||
|
||||
## [2.11.0] - 2020-03-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Upgrade Electron to v8
|
||||
- Upgrade Angular to v9
|
||||
- App settings are now portable in a json file
|
||||
- General code source maintenance
|
||||
|
||||
### Added
|
||||
|
||||
- Added Support for pixroute.com
|
||||
- Added grab post title from thread option
|
||||
|
||||
## [2.10.8] - 2020-01-25
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix app url
|
||||
|
||||
## [2.10.7] - 2020-01-25
|
||||
|
||||
### Changed
|
||||
|
||||
- Drop SockJs and switch to WebSocket
|
||||
- Autoreconnect front to back
|
||||
|
||||
## [2.10.6] - 2020-01-19
|
||||
|
||||
### Changed
|
||||
|
||||
- Limit cache size
|
||||
|
||||
## [2.10.5] - 2020-01-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [2.10.4] - 2020-01-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Zero value for total max download will disable the limit
|
||||
- Add system notification
|
||||
- Bug fixes
|
||||
|
||||
## [2.10.3] - 2020-01-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Some bug fixes
|
||||
- Enhance download queue logic
|
||||
- Clear cache button
|
||||
- Fix the partial status
|
||||
- Sanitize URLs with spaces
|
||||
|
||||
### Added
|
||||
|
||||
- Support for Postimg host
|
||||
- Support for Imagevenue host
|
||||
|
||||
## [2.10.2] - 2019-12-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Upgrade electron to v7
|
||||
- Bug fix
|
||||
|
||||
## [2.10.1] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Bug fix
|
||||
|
||||
## [2.10.0] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix spring dependency bug
|
||||
|
||||
### Added
|
||||
|
||||
- Add support for imgspice
|
||||
|
||||
## [2.9.0] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix build issue
|
||||
|
||||
## [2.9.0] - 2019-12-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Decrease timeout for fast fail
|
||||
|
||||
### Added
|
||||
|
||||
- Allow many hosts to run at the same time
|
||||
- Add title to gallery
|
||||
- Add global concurrent downloads
|
||||
- Add counter badge in link collector
|
||||
|
||||
## [2.8.1] - 2019-12-24
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix issue with path creation
|
||||
|
||||
## [2.8.0] - 2019-12-24
|
||||
|
||||
### Added
|
||||
|
||||
- Photo gallery feature
|
||||
|
||||
## [2.7.1] - 2019-12-21
|
||||
|
||||
### Added
|
||||
|
||||
- Add version to title bar
|
||||
|
||||
## [2.7.0] - 2019-12-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Add missing icon
|
||||
|
||||
### Added
|
||||
|
||||
- Add support for pixxxels
|
||||
|
||||
## [2.6.0] - 2019-12-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Disable download button when nothing selected in Link collector grab menu
|
||||
- Better quality app icon
|
||||
- Implement context menu on download tab
|
||||
- Add stop remove and start button to context menu
|
||||
- Link will not be removed automatically from link collector tab
|
||||
- Some UI fixes
|
||||
|
||||
## [2.5.1] - 2019-12-09
|
||||
|
||||
### Changed
|
||||
|
||||
- Disable download button when nothing selected in Link collector grab menu
|
||||
|
||||
### Added
|
||||
|
||||
- Add hosts column in Link collector grab menu
|
||||
|
||||
## [2.5.0] - 2019-12-08
|
||||
|
||||
### Changed
|
||||
|
||||
- Change loading screen
|
||||
- UI Fixes
|
||||
- Fix bug with previews
|
||||
|
||||
### Added
|
||||
|
||||
- Add link collector feature
|
||||
- Add support for multiple links at a time
|
||||
|
||||
## [2.4.1] - 2019-11-26
|
||||
|
||||
### Changed
|
||||
|
||||
- Increase attempts for establishing ws connection between front and back end
|
||||
- Change default download folder to Home folder (installer), App folder (portable)
|
||||
|
||||
## [2.4.0] - 2019-11-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Pimpandhost Host support
|
||||
|
||||
## [2.3.2] - 2019-11-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a bug with the scan component
|
||||
|
||||
## [2.3.1] - 2019-11-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix connexion reset issue
|
||||
- Fix Post Status when autostart option is selected
|
||||
- Add stop all button
|
||||
|
||||
## [2.3.0] - 2019-11-21
|
||||
|
||||
### Changed
|
||||
|
||||
- UI enhancements and overhaul
|
||||
- Stability fixes with HTTP connections
|
||||
- MacOS official support
|
||||
- Use local sockjs and material icons font
|
||||
|
||||
## [2.2.5] - 2019-11-09
|
||||
|
||||
### Changed
|
||||
|
||||
- Exclude thumbs download for imagezilla and pixhost
|
||||
- Change retry on fail logic (may help avoid having download errors)
|
||||
|
||||
## [2.2.4] - 2019-10-05
|
||||
|
||||
### Added
|
||||
|
||||
- AppImage target for linux
|
||||
|
||||
## [2.2.3] - 2019-10-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Strip libjvm.so for debian
|
||||
|
||||
## [2.2.2] - 2019-10-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Update product name
|
||||
|
||||
## [2.2.1] - 2019-10-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Update dependency list for deb package
|
||||
|
||||
## [2.2.0] - 2019-10-01
|
||||
|
||||
### Added
|
||||
|
||||
- Embedd JRE within the app
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix 'Start' on pending items
|
||||
- Upgrade to Java 11
|
||||
|
||||
## [2.1.0] - 2019-09-30
|
||||
|
||||
### Added
|
||||
|
||||
- Add a create a subfolder per thread option
|
||||
|
||||
## [2.0.4] - 2019-09-27
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix some issues with ImxHost
|
||||
|
||||
## [2.0.3] - 2019-09-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Update deb dependencies
|
||||
|
||||
## [2.0.2] - 2019-09-23
|
||||
|
||||
### Changed
|
||||
|
||||
- Add portable mode
|
||||
|
||||
## [2.0.1] - 2019-09-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Change app name
|
||||
|
||||
## [2.0.0] - 2019-09-22
|
||||
|
||||
### Added
|
||||
|
||||
- Add sub folder option
|
||||
- Add clear finished option
|
||||
|
||||
### Changed
|
||||
|
||||
- UI enhancements
|
||||
- Bug fixes
|
||||
- Change data and log paths
|
||||
|
||||
## [1.6.5] - 2019-09-18
|
||||
|
||||
### Changed
|
||||
|
||||
- electron update
|
||||
|
||||
## [1.6.4] - 2019-09-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Change menu item name
|
||||
|
||||
## [1.6.3] - 2019-09-18
|
||||
|
||||
### Changed
|
||||
|
||||
- Better multithread management
|
||||
- Fix partial state UI
|
||||
|
||||
## [1.6.2] - 2019-09-17
|
||||
|
||||
### Changed
|
||||
|
||||
- Rework UI colors
|
||||
- Fix scan input bug
|
||||
|
||||
## [1.6.1] - 2019-09-16
|
||||
|
||||
### Changed
|
||||
|
||||
- Rework UI
|
||||
- Fix clipboard monitoring
|
||||
|
||||
## [1.6.0] - 2019-09-15
|
||||
|
||||
### Changed
|
||||
|
||||
- Rework the UI
|
||||
- Add a dark theme
|
||||
- Fix the force ordering bug
|
||||
- Other bug fixes
|
||||
|
||||
## [1.5.2] - 2019-09-01
|
||||
|
||||
### Added
|
||||
|
||||
- imx.to host fix
|
||||
|
||||
## [1.5.1] - 2019-08-24
|
||||
|
||||
### Added
|
||||
|
||||
- Bug fixes
|
||||
|
||||
## [1.5.0] - 2019-08-24
|
||||
|
||||
### Added
|
||||
|
||||
- Display logged in user
|
||||
- Add post details when clicking on main list items (Status, Post URL, Host)
|
||||
- Add open download location on main list items menu
|
||||
@@ -121,87 +637,127 @@
|
||||
- Add link to image from details list
|
||||
|
||||
## [1.4.10] - 2019-08-15
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a rename issue on windows
|
||||
|
||||
## [1.4.9] - 2019-08-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Update Built-By to be generic
|
||||
|
||||
## [1.4.8] - 2019-08-13
|
||||
|
||||
### Changed
|
||||
|
||||
- Add option to force image ordering
|
||||
|
||||
## [1.4.7] - 2019-08-13
|
||||
|
||||
### Changed
|
||||
- Fixed a bug in image filename extensions
|
||||
|
||||
- Fixed a bug in image filename extensions
|
||||
|
||||
## [1.4.6] - 2019-08-06
|
||||
|
||||
### Changed
|
||||
|
||||
- Fixed a bug when creating image file names
|
||||
- Remove unnecessary logs
|
||||
- Fix data.json init
|
||||
- Fix data.json init
|
||||
|
||||
## [1.4.5] - 2019-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Add min width and height for electron app
|
||||
|
||||
### Added
|
||||
|
||||
- Add download speed to status bar
|
||||
|
||||
## [1.4.4] - 2019-08-01
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix names for ImgboxHost
|
||||
- Enhance UI
|
||||
- Minor fixes
|
||||
|
||||
### Added
|
||||
|
||||
- Add start and stop button
|
||||
|
||||
## [1.4.3] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Update some messages
|
||||
- Fix scrolling issue
|
||||
|
||||
## [1.4.2] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix layout issue in web mode
|
||||
|
||||
## [1.4.1] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Change title's font
|
||||
|
||||
### Added
|
||||
|
||||
- Add borders
|
||||
|
||||
## [1.4.0] - 2019-07-14
|
||||
|
||||
### Changed
|
||||
|
||||
- Better error handling on startup
|
||||
- Better support for linux AppImage
|
||||
|
||||
### Added
|
||||
|
||||
- Clear all completed
|
||||
- Add clipboard support
|
||||
- Add option to remove all posts
|
||||
- Add confirmation on remove
|
||||
|
||||
## [1.3.0-rc0] - 2019-07-04
|
||||
|
||||
### Changed
|
||||
|
||||
- Change default setting for autostart false -> true
|
||||
- Change default setting for max concurrent downloads 1 -> 4
|
||||
|
||||
### Added
|
||||
|
||||
- Desktop app built with electron
|
||||
- Add the option to remove a post
|
||||
|
||||
## [1.2.0] - 2019-06-23
|
||||
|
||||
### Added
|
||||
|
||||
- Add a shutdown button to gracefully shutdown the app
|
||||
- Open and redirect the browser to the app address on startup
|
||||
- Add the option to run the app in headless mode (-Djava.awt.headless=true)
|
||||
|
||||
## [1.1.1] - 2019-06-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Fix a stability issue with progress updates when the app is runninh for a long time
|
||||
|
||||
## [1.0.0] - 2019-06-22
|
||||
|
||||
### Added
|
||||
|
||||
- Add real time communication for progress updates via websocket (SockJS)
|
||||
- Add settings for download path, max concurrent download, enable/disable integration with vipergirls
|
||||
- Add integration with vipergitls.to (possibility to auto leave thanks after grab post)
|
||||
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
|
||||
- Add support for acidimg.cc, imagezilla.net, imgbox.com, imx.to, pixhost.to, turboimagehost.com
|
||||
|
||||
@@ -1,26 +1,19 @@
|
||||
# Viper Ripper!
|
||||
# VRipper!
|
||||
|
||||
This is my spin for a gallery ripper app for [vipergirls](https://vipergirls.to) website.
|
||||
The purpose of this project is to build a robust and clean application to conveniently download photo galleries using modern web technologies, Java + Spring boot for the back end and angular + electron for the front end.
|
||||
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to).
|
||||
|
||||
## How to build
|
||||
You need a recent version of maven 3.6.1+.
|
||||
|
||||
The application support 2 build modes:
|
||||
- Desktop app (*Partially cross platform): Self contained app, built with electron, depends on Java runtimes.
|
||||
- Server app (Fully cross platform): Depends on Java runtimes, you need a web browser to access the app UI.
|
||||
You need JDK 17 and a recent version of maven 3.6.1+
|
||||
|
||||
Most people will be interested only on the desktop app, however if you have a nas or a server, the server app is there for you.
|
||||
|
||||
To build the server app:
|
||||
To build run the following:
|
||||
|
||||
mvn clean install
|
||||
To build the desktop app:
|
||||
|
||||
mvn clean install -Pelectron
|
||||
Build artifact is located under
|
||||
|
||||
Maven will automatically handle front end compilation. However, for development, you will need to install a recent version of nodejs on your system.
|
||||
vripper-project\vripper-gui\target\vripper-gui-4.0.0.jar
|
||||
|
||||
*Partially cross platform:
|
||||
Technically cross platform, but the Desktop app relies on electron builder to make appropriate binaries for each platform. mac binaries can only be generated on a mac, and i don't own one. So i can only provide binaries for windows and linux.
|
||||
The Server app however depends only on Java, so it will work just fine on a mac, you only need a browser to access it.
|
||||
Copy the artifact into any other folder and run:
|
||||
|
||||
java -jar vripper-gui-4.0.0.jar
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>tn.mnlr</groupId>
|
||||
<artifactId>vripper</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<packaging>pom</packaging>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.1.1.RELEASE</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
</parent>
|
||||
<modules>
|
||||
<module>vripper-server</module>
|
||||
<module>vripper-ui</module>
|
||||
<module>vripper-electron</module>
|
||||
</modules>
|
||||
</project>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>me.mnlr</groupId>
|
||||
<artifactId>vripper-project</artifactId>
|
||||
<modules>
|
||||
<module>vripper-core</module>
|
||||
<module>vripper-web-ui</module>
|
||||
<module>vripper-web</module>
|
||||
<module>vripper-gui</module>
|
||||
</modules>
|
||||
<packaging>pom</packaging>
|
||||
<version>4.1.0</version>
|
||||
<properties>
|
||||
<maven.deploy.skip>true</maven.deploy.skip>
|
||||
</properties>
|
||||
</project>
|
||||
|
||||
Vendored
@@ -0,0 +1,128 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<relativePath/>
|
||||
<version>3.0.6</version>
|
||||
</parent>
|
||||
<groupId>me.mnlr.vripper</groupId>
|
||||
<artifactId>vripper-core</artifactId>
|
||||
<version>4.1.0</version>
|
||||
<name>vripper-core</name>
|
||||
<description>vripper-core</description>
|
||||
<properties>
|
||||
<java.version>17</java.version>
|
||||
<kotlin.version>1.8.21</kotlin.version>
|
||||
<maven.deploy.skip>false</maven.deploy.skip>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<artifactId>kotlin-reflect</artifactId>
|
||||
<groupId>org.jetbrains.kotlin</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>kotlin-stdlib</artifactId>
|
||||
<groupId>org.jetbrains.kotlin</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>jackson-module-kotlin</artifactId>
|
||||
<groupId>com.fasterxml.jackson.module</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>spring-boot-starter-jdbc</artifactId>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>liquibase-core</artifactId>
|
||||
<groupId>org.liquibase</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>hsqldb</artifactId>
|
||||
<groupId>org.hsqldb</groupId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>htmlcleaner</artifactId>
|
||||
<groupId>net.sourceforge.htmlcleaner</groupId>
|
||||
<version>2.26</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>reactor-core</artifactId>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>failsafe</artifactId>
|
||||
<groupId>net.jodah</groupId>
|
||||
<version>2.4.4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>caffeine</artifactId>
|
||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>jackson-dataformat-yaml</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>spring-restdocs-mockmvc</artifactId>
|
||||
<groupId>org.springframework.restdocs</groupId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<sourceDirectory>${project.basedir}/src/main/kotlin</sourceDirectory>
|
||||
<testSourceDirectory>${project.basedir}/src/test/kotlin</testSourceDirectory>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jetbrains.kotlin</groupId>
|
||||
<artifactId>kotlin-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<args>
|
||||
<arg>-Xjsr305=strict</arg>
|
||||
</args>
|
||||
<compilerPlugins>
|
||||
<plugin>spring</plugin>
|
||||
</compilerPlugins>
|
||||
</configuration>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<artifactId>kotlin-maven-allopen</artifactId>
|
||||
<groupId>org.jetbrains.kotlin</groupId>
|
||||
<version>${kotlin.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<distributionManagement>
|
||||
<repository>
|
||||
<id>github</id>
|
||||
<name>GitHub death-claw Apache Maven Packages</name>
|
||||
<url>https://maven.pkg.github.com/death-claw/vripper-project</url>
|
||||
</repository>
|
||||
</distributionManagement>
|
||||
</project>
|
||||
@@ -0,0 +1,149 @@
|
||||
package me.mnlr.vripper
|
||||
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.DownloadService
|
||||
import me.mnlr.vripper.download.PostDownloadRunnable
|
||||
import me.mnlr.vripper.exception.PostParseException
|
||||
import me.mnlr.vripper.model.PostItem
|
||||
import me.mnlr.vripper.repositories.LogEventRepository
|
||||
import me.mnlr.vripper.repositories.ThreadRepository
|
||||
import me.mnlr.vripper.services.*
|
||||
import me.mnlr.vripper.tasks.ThreadLookupRunnable
|
||||
import org.springframework.stereotype.Service
|
||||
import java.util.*
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@Service
|
||||
class AppEndpointService(
|
||||
private val downloadService: DownloadService,
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val threadRepository: ThreadRepository,
|
||||
private val threadCacheService: ThreadCacheService,
|
||||
private val eventRepository: LogEventRepository,
|
||||
private val asyncTaskRunnerService: AsyncTaskRunnerService,
|
||||
private val settingsService: SettingsService
|
||||
) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Synchronized
|
||||
fun scanLinks(postLinks: String) {
|
||||
if (postLinks.isBlank()) {
|
||||
log.warn("Nothing to scan")
|
||||
return
|
||||
}
|
||||
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isEmpty() }
|
||||
.map { it.trim() }.filter { it.isNotEmpty() }
|
||||
for (link in urlList) {
|
||||
log.debug("Starting to process thread: $link")
|
||||
if (!link.startsWith(settingsService.settings.viperSettings.host)) {
|
||||
continue
|
||||
}
|
||||
var threadId: String
|
||||
var postId: String?
|
||||
val m = Pattern.compile(
|
||||
Pattern.quote(settingsService.settings.viperSettings.host) + "/threads/(\\d+)((.*p=)(\\d+))?"
|
||||
).matcher(link)
|
||||
if (m.find()) {
|
||||
threadId = m.group(1)
|
||||
postId = m.group(4)
|
||||
if (postId == null) {
|
||||
asyncTaskRunnerService.sink.emitNext(
|
||||
ThreadLookupRunnable(
|
||||
threadId, settingsService.settings
|
||||
)
|
||||
) { _, _ -> true }
|
||||
} else {
|
||||
asyncTaskRunnerService.sink.emitNext(
|
||||
PostDownloadRunnable(
|
||||
threadId, postId
|
||||
)
|
||||
) { _, _ -> true }
|
||||
}
|
||||
} else {
|
||||
log.error("Cannot retrieve thread id from URL $link")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun restartAll(posIds: List<String> = listOf()) {
|
||||
downloadService.restartAll(posIds)
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun download(posts: List<Pair<String, String>>) {
|
||||
for (post in posts) {
|
||||
asyncTaskRunnerService.sink.emitNext(
|
||||
PostDownloadRunnable(
|
||||
post.first, post.second
|
||||
)
|
||||
) { _, _ -> true }
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun stopAll(postIdList: List<String>?) {
|
||||
downloadService.stopAll(postIdList)
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun remove(postIdList: List<String>) {
|
||||
downloadService.stopAll(postIdList)
|
||||
dataTransaction.removeAll(postIdList)
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun clearCompleted(): List<String> {
|
||||
return dataTransaction.clearCompleted()
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
@Throws(PostParseException::class)
|
||||
fun grab(threadId: String): List<PostItem> {
|
||||
return try {
|
||||
val thread = threadRepository.findByThreadId(threadId).orElseThrow {
|
||||
PostParseException(
|
||||
String.format(
|
||||
"Unable to find links for threadId = %s", threadId
|
||||
)
|
||||
)
|
||||
}
|
||||
val threadLookupResult = threadCacheService[thread.threadId]
|
||||
threadLookupResult.postItemList.ifEmpty {
|
||||
log.error(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s", threadId
|
||||
)
|
||||
)
|
||||
throw PostParseException(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s", threadId
|
||||
)
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
throw PostParseException(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s, %s", threadId, e.message
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun threadRemove(threadIdList: List<String>) {
|
||||
threadIdList.forEach {
|
||||
dataTransaction.removeThread(it)
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun threadClear() {
|
||||
dataTransaction.clearQueueLinks()
|
||||
}
|
||||
|
||||
fun logClear() {
|
||||
eventRepository.deleteAll()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package me.mnlr.vripper
|
||||
|
||||
import org.springframework.context.event.ContextRefreshedEvent
|
||||
import org.springframework.context.event.EventListener
|
||||
import org.springframework.stereotype.Component
|
||||
import me.mnlr.vripper.repositories.PostDownloadStateRepository
|
||||
import me.mnlr.vripper.services.DataTransaction
|
||||
|
||||
@Component
|
||||
class EventListenerBean(
|
||||
private val postDownloadStateRepository: PostDownloadStateRepository,
|
||||
private val dataTransaction: DataTransaction
|
||||
) {
|
||||
@EventListener
|
||||
fun onApplicationEvent(event: ContextRefreshedEvent?) {
|
||||
postDownloadStateRepository.setDownloadingToStopped()
|
||||
dataTransaction.sortPostsByRank()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package me.mnlr.vripper
|
||||
|
||||
import jakarta.annotation.PreDestroy
|
||||
import org.springframework.beans.factory.annotation.Value
|
||||
import org.springframework.jdbc.core.JdbcTemplate
|
||||
import org.springframework.stereotype.Component
|
||||
import me.mnlr.vripper.download.DownloadService
|
||||
import me.mnlr.vripper.services.AsyncTaskRunnerService
|
||||
import java.nio.file.Path
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.*
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@Component
|
||||
class Management(
|
||||
private val jdbcTemplate: JdbcTemplate,
|
||||
@Value("\${base.dir}") baseDir: String,
|
||||
@Value("\${base.dir.name}") baseDirName: String,
|
||||
private val asyncTaskRunnerService: AsyncTaskRunnerService,
|
||||
private val downloadService: DownloadService
|
||||
) {
|
||||
private val backupFolder: Path
|
||||
|
||||
init {
|
||||
backupFolder = Path.of(baseDir, baseDirName, "backup")
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
fun destroy() {
|
||||
// taskRunnerService.destroy()
|
||||
downloadService.destroy()
|
||||
jdbcTemplate.execute("SHUTDOWN")
|
||||
}
|
||||
|
||||
// @PostConstruct
|
||||
// @Scheduled(cron = "0 0 0 ? * *")
|
||||
// private fun backup() {
|
||||
//
|
||||
// for (file in Files.list(backupFolder)) {
|
||||
// val matcher = BACKUP_FILE_PATTERN.matcher(file.fileName.toString())
|
||||
// if (matcher.find()) {
|
||||
// val localDate = LocalDate.parse(matcher.group(1), FORMATTER)
|
||||
// if (localDate.isEqual(LocalDate.now())) {
|
||||
// return
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // create a backup file
|
||||
// val backupFile = backupFolder.resolve("db_${LocalDateTime.now().format(FORMATTER)}.tar.gz")
|
||||
// jdbcTemplate.execute("BACKUP DATABASE TO '${backupFile}' BLOCKING")
|
||||
// }
|
||||
|
||||
companion object {
|
||||
val FORMATTER: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd")
|
||||
private val BACKUP_FILE_PATTERN = Pattern.compile("^db_(\\d{4}-\\d{2}-\\d{2})\\.tar\\.gz$")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package me.mnlr.vripper
|
||||
|
||||
import org.springframework.beans.BeansException
|
||||
import org.springframework.context.ApplicationContext
|
||||
import org.springframework.context.ApplicationContextAware
|
||||
import org.springframework.context.ConfigurableApplicationContext
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.stereotype.Component
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
|
||||
@Component
|
||||
@EnableScheduling
|
||||
class SpringContext : ApplicationContextAware {
|
||||
@Throws(BeansException::class)
|
||||
override fun setApplicationContext(context: ApplicationContext) {
|
||||
|
||||
// store ApplicationContext reference to access required beans later on
|
||||
Companion.context = context as ConfigurableApplicationContext
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val log by LoggerDelegate()
|
||||
private lateinit var context: ConfigurableApplicationContext
|
||||
fun <T> getBean(beanClass: Class<T>): T {
|
||||
return context.getBean(beanClass)
|
||||
}
|
||||
|
||||
fun <T> getBeansOfType(beanClass: Class<T>): Map<String, T> {
|
||||
return context.getBeansOfType(beanClass)
|
||||
}
|
||||
|
||||
fun close() {
|
||||
log.info("Application terminating...")
|
||||
context.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package me.mnlr.vripper
|
||||
|
||||
import java.io.PrintWriter
|
||||
import java.io.StringWriter
|
||||
|
||||
fun Throwable.formatToString(): String {
|
||||
return StringWriter().use { stringWriter ->
|
||||
PrintWriter(stringWriter).use { printWriter ->
|
||||
this.printStackTrace(printWriter)
|
||||
printWriter.flush()
|
||||
stringWriter.flush()
|
||||
stringWriter.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun Long.formatSI(): String {
|
||||
return humanReadableByteCount(this, false)
|
||||
}
|
||||
|
||||
private fun humanReadableByteCount(bytes: Long, si: Boolean): String {
|
||||
val unit = if (si) 1000 else 1024
|
||||
if (bytes < unit) return "$bytes B"
|
||||
val exp = (Math.log(bytes.toDouble()) / Math.log(unit.toDouble())).toInt()
|
||||
val pre = (if (si) "kMGTPE" else "KMGTPE")[exp - 1].toString() + if (si) "" else "i"
|
||||
return String.format("%.1f %sB", bytes / Math.pow(unit.toDouble(), exp.toDouble()), pre)
|
||||
}
|
||||
|
||||
|
||||
fun getExtension(fileName: String): String {
|
||||
return if (fileName.contains(".")) fileName.substring(fileName.lastIndexOf(".") + 1) else ""
|
||||
}
|
||||
|
||||
fun getFileNameWithoutExtension(fileName: String): String {
|
||||
return if (fileName.contains(".")) fileName.substring(
|
||||
0,
|
||||
fileName.lastIndexOf(".")
|
||||
) else fileName
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package me.mnlr.vripper.delegate
|
||||
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
import kotlin.properties.ReadOnlyProperty
|
||||
import kotlin.reflect.KProperty
|
||||
|
||||
class LoggerDelegate : ReadOnlyProperty<Any?, Logger> {
|
||||
private var logger: Logger? = null
|
||||
override operator fun getValue(thisRef: Any?, property: KProperty<*>): Logger {
|
||||
if (logger == null) {
|
||||
logger = LoggerFactory.getLogger(thisRef!!.javaClass)
|
||||
}
|
||||
return logger!!
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package me.mnlr.vripper.download
|
||||
|
||||
import jakarta.annotation.PostConstruct
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.entities.ImageDownloadState
|
||||
import me.mnlr.vripper.entities.LogEvent
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import me.mnlr.vripper.formatToString
|
||||
import me.mnlr.vripper.host.Host
|
||||
import me.mnlr.vripper.repositories.ImageRepository
|
||||
import me.mnlr.vripper.repositories.LogEventRepository
|
||||
import me.mnlr.vripper.repositories.PostDownloadStateRepository
|
||||
import me.mnlr.vripper.services.*
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.RetryPolicy
|
||||
import org.springframework.beans.factory.annotation.Value
|
||||
import org.springframework.stereotype.Service
|
||||
import java.util.*
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import java.util.stream.Collectors
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
@Service
|
||||
class DownloadService(
|
||||
@param:Value("\${download.pool-size}") private val maxPoolSize: Int,
|
||||
private val settingsService: SettingsService,
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val hosts: List<Host>,
|
||||
private val eventRepository: LogEventRepository,
|
||||
private val retryPolicyService: RetryPolicyService,
|
||||
private val postDownloadStateRepository: PostDownloadStateRepository,
|
||||
private val imageRepository: ImageRepository,
|
||||
private val asyncTaskRunnerService: AsyncTaskRunnerService,
|
||||
) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
// Class fields
|
||||
private val executor: ExecutorService = Executors.newFixedThreadPool(maxPoolSize)
|
||||
private val running: MutableMap<Host, MutableList<ImageDownloadRunnable>> = mutableMapOf()
|
||||
private val pending: MutableMap<Host, MutableList<ImageDownloadRunnable>> = mutableMapOf()
|
||||
private val lock = ReentrantLock()
|
||||
private val condition = lock.newCondition()
|
||||
private val pollThread: Thread
|
||||
|
||||
init {
|
||||
pollThread = Thread(
|
||||
{
|
||||
val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
|
||||
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
|
||||
while (!Thread.interrupted()) {
|
||||
lock.withLock {
|
||||
candidates.addAll(getCandidates(candidateCount()))
|
||||
candidates.forEach {
|
||||
if (canRun(it.context.image.host)) {
|
||||
accepted.add(it)
|
||||
running[it.context.image.host]!!.add(it)
|
||||
log.debug("${it.context.image.url} accepted to run")
|
||||
}
|
||||
}
|
||||
accepted.forEach {
|
||||
pending[it.context.image.host]?.remove(it)
|
||||
schedule(it)
|
||||
}
|
||||
accepted.clear()
|
||||
candidates.clear()
|
||||
condition.await()
|
||||
}
|
||||
}
|
||||
}, "Download scheduler thread"
|
||||
)
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
fun init() {
|
||||
pollThread.start()
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
log.info("Shutting down ExecutionService")
|
||||
pollThread.interrupt()
|
||||
executor.shutdown()
|
||||
stop(postDownloadStateRepository.findAll().map { it.postId })
|
||||
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
|
||||
log.warn("Some jobs are still running!, forcing shutdown")
|
||||
executor.shutdownNow()
|
||||
}
|
||||
}
|
||||
|
||||
fun stopAll(postIds: List<String>?) {
|
||||
if (postIds != null) {
|
||||
stop(postIds)
|
||||
} else {
|
||||
stop(postDownloadStateRepository.findAll().map(PostDownloadState::postId))
|
||||
}
|
||||
}
|
||||
|
||||
fun restartAll(postIds: List<String> = listOf()) {
|
||||
if (postIds.isNotEmpty()) {
|
||||
restart(postIds)
|
||||
} else {
|
||||
restart(postDownloadStateRepository.findAll().map(PostDownloadState::postId))
|
||||
}
|
||||
}
|
||||
|
||||
private fun restart(postIds: List<String>) {
|
||||
lock.withLock {
|
||||
val data: MutableMap<PostDownloadState, Collection<ImageDownloadState>> = mutableMapOf()
|
||||
for (postId in postIds) {
|
||||
if (isPending(postId)) {
|
||||
log.warn("Cannot restart, jobs are currently running for post id $postIds")
|
||||
continue
|
||||
}
|
||||
val imageDownloadStates: List<ImageDownloadState> =
|
||||
imageRepository.findByPostIdAndIsNotCompleted(postId)
|
||||
if (imageDownloadStates.isEmpty()) {
|
||||
continue
|
||||
}
|
||||
val postDownloadState: PostDownloadState =
|
||||
postDownloadStateRepository.findByPostId(postId).orElseThrow()
|
||||
log.debug("Restarting ${imageDownloadStates.size} jobs for post id $postIds")
|
||||
postDownloadState.status = Status.PENDING
|
||||
dataTransaction.update(postDownloadState)
|
||||
data[postDownloadState] = imageDownloadStates
|
||||
}
|
||||
|
||||
for ((postDownloadState, imageDownloadStates) in data) {
|
||||
postDownloadState.status = Status.PENDING
|
||||
dataTransaction.update(postDownloadState)
|
||||
for (imageDownloadState in imageDownloadStates) {
|
||||
log.debug("Enqueuing a job for ${imageDownloadState.url}")
|
||||
with(imageDownloadState) {
|
||||
this.status = Status.STOPPED
|
||||
this.current = 0
|
||||
}
|
||||
dataTransaction.update(imageDownloadState)
|
||||
val imageDownloadRunnable = ImageDownloadRunnable(
|
||||
imageDownloadState.id!!, settingsService.settings
|
||||
)
|
||||
pending.computeIfAbsent(
|
||||
imageDownloadState.host
|
||||
) { mutableListOf() }
|
||||
pending[imageDownloadState.host]!!.add(imageDownloadRunnable)
|
||||
}
|
||||
}
|
||||
condition.signal()
|
||||
}
|
||||
}
|
||||
|
||||
private fun isPending(postId: String): Boolean {
|
||||
lock.withLock {
|
||||
return pending.values.flatten().any { it.context.image.postId == postId }
|
||||
}
|
||||
}
|
||||
|
||||
private fun isRunning(postId: String): Boolean {
|
||||
lock.withLock {
|
||||
return running.values.flatten().any { it.context.image.postId == postId }
|
||||
}
|
||||
}
|
||||
|
||||
private fun stop(postIds: List<String>) {
|
||||
lock.withLock {
|
||||
for (postId in postIds) {
|
||||
val postDownloadState: PostDownloadState =
|
||||
postDownloadStateRepository.findByPostId(postId).orElseThrow()
|
||||
pending.values.forEach { pending ->
|
||||
pending.removeIf { it.context.image.postId == postId }
|
||||
}
|
||||
running.values.flatten()
|
||||
.filter { p: ImageDownloadRunnable -> p.context.image.postId == postId }
|
||||
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
|
||||
dataTransaction.stopImagesByPostIdAndIsNotCompleted(postId)
|
||||
dataTransaction.finishPost(postDownloadState)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun canRun(host: Host): Boolean {
|
||||
val totalRunning = running.values.sumOf { it.size }
|
||||
return (running[host]!!.size < settingsService.settings.connectionSettings.maxThreads && if (settingsService.settings.connectionSettings.maxTotalThreads == 0) totalRunning < maxPoolSize else totalRunning < settingsService.settings.connectionSettings.maxTotalThreads)
|
||||
}
|
||||
|
||||
private fun candidateCount(): Map<Host, Int> {
|
||||
val map: MutableMap<Host, Int> = mutableMapOf()
|
||||
hosts.forEach { h: Host ->
|
||||
val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
|
||||
h
|
||||
) { mutableListOf() }
|
||||
val count: Int =
|
||||
settingsService.settings.connectionSettings.maxThreads - imageDownloadRunnableList.size
|
||||
log.debug("Download slots for ${h.host}: $count")
|
||||
map[h] = count
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
private fun getCandidates(candidateCount: Map<Host, Int>): List<ImageDownloadRunnable> {
|
||||
val hostIntegerMap: MutableMap<Host, Int> = candidateCount.toMutableMap()
|
||||
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
|
||||
hosts@ for (host in pending.keys) {
|
||||
val list: List<ImageDownloadRunnable> =
|
||||
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.context.post.rank }
|
||||
.thenComparingInt { it.context.image.index })
|
||||
for (imageDownloadRunnable in list) {
|
||||
val count = hostIntegerMap[host] ?: 0
|
||||
if (count > 0) {
|
||||
candidates.add(imageDownloadRunnable)
|
||||
hostIntegerMap[host] = count - 1
|
||||
} else {
|
||||
continue@hosts
|
||||
}
|
||||
}
|
||||
}
|
||||
if (log.isDebugEnabled) {
|
||||
val collect: Map<Host, List<ImageDownloadRunnable>> =
|
||||
candidates.stream().collect(Collectors.groupingBy { it.context.image.host })
|
||||
collect.forEach {
|
||||
log.debug(
|
||||
"Candidate download for ${it.key.host} ${it.value.size}/${candidateCount[it.key]}"
|
||||
)
|
||||
}
|
||||
}
|
||||
return candidates.sortedWith(Comparator.comparing { v: ImageDownloadRunnable -> v.context.post.rank })
|
||||
}
|
||||
|
||||
private fun schedule(imageDownloadRunnable: ImageDownloadRunnable) {
|
||||
log.debug("Scheduling a job for ${imageDownloadRunnable.context.image.url}")
|
||||
executor.execute {
|
||||
try {
|
||||
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload())
|
||||
.onFailure {
|
||||
try {
|
||||
eventRepository.save(
|
||||
LogEvent(
|
||||
type = LogEvent.Type.DOWNLOAD,
|
||||
status = LogEvent.Status.ERROR,
|
||||
message = "Failed to download ${imageDownloadRunnable.context.image.url}\n ${it.failure.formatToString()}"
|
||||
)
|
||||
)
|
||||
} catch (exp: Exception) {
|
||||
log.error("Failed to save event", exp)
|
||||
}
|
||||
log.error(
|
||||
"Failed to download ${imageDownloadRunnable.context.image.url} after ${it.attemptCount} tries",
|
||||
it.failure
|
||||
)
|
||||
val image = imageDownloadRunnable.context.image
|
||||
image.status = Status.ERROR
|
||||
dataTransaction.update(image)
|
||||
}.onComplete {
|
||||
|
||||
afterJobFinish(imageDownloadRunnable)
|
||||
log.debug(
|
||||
"Finished downloading ${imageDownloadRunnable.context.image.url}"
|
||||
)
|
||||
}.run(imageDownloadRunnable::run)
|
||||
} catch (ignored: Exception) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
|
||||
lock.withLock {
|
||||
running[imageDownloadRunnable.context.image.host]!!.remove(imageDownloadRunnable)
|
||||
if (!isPending(imageDownloadRunnable.context.image.postId) && !isRunning(
|
||||
imageDownloadRunnable.context.image.postId
|
||||
)
|
||||
) {
|
||||
dataTransaction.finishPost(imageDownloadRunnable.context.post)
|
||||
}
|
||||
condition.signal()
|
||||
}
|
||||
}
|
||||
|
||||
fun pendingCount(): Int {
|
||||
return pending.values.sumOf { it.size }
|
||||
}
|
||||
|
||||
fun runningCount(): Int {
|
||||
return running.values.sumOf { it.size }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package me.mnlr.vripper.download
|
||||
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.entities.ImageDownloadState
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.repositories.ImageRepository
|
||||
import me.mnlr.vripper.repositories.PostDownloadStateRepository
|
||||
|
||||
class ImageDownloadContext(val imageId: Long) {
|
||||
|
||||
private val imageRepository: ImageRepository =
|
||||
SpringContext.getBean(ImageRepository::class.java)
|
||||
private val postDownloadStateRepository: PostDownloadStateRepository =
|
||||
SpringContext.getBean(PostDownloadStateRepository::class.java)
|
||||
|
||||
|
||||
val httpContext: HttpClientContext = HttpClientContext.create()
|
||||
val postId = image.postIdRef
|
||||
var stopped = false
|
||||
val image: ImageDownloadState
|
||||
get() = imageRepository.findById(imageId).orElseThrow()
|
||||
val post: PostDownloadState
|
||||
get() = postDownloadStateRepository.findById(postId).orElseThrow()
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package me.mnlr.vripper.download
|
||||
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.entities.ImageDownloadState
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import me.mnlr.vripper.exception.DownloadException
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.getExtension
|
||||
import me.mnlr.vripper.host.DownloadedImage
|
||||
import me.mnlr.vripper.host.ImageMimeType
|
||||
import me.mnlr.vripper.model.Settings
|
||||
import me.mnlr.vripper.services.*
|
||||
import net.jodah.failsafe.function.CheckedRunnable
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.impl.client.BasicCookieStore
|
||||
import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.util.*
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
class ImageDownloadRunnable(
|
||||
private val imageInternalId: Long, private val settings: Settings
|
||||
) : CheckedRunnable {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
private val dataTransaction: DataTransaction = SpringContext.getBean(DataTransaction::class.java)
|
||||
private val pathService: PathService = SpringContext.getBean(PathService::class.java)
|
||||
|
||||
val context: ImageDownloadContext = ImageDownloadContext(imageInternalId)
|
||||
private val image: ImageDownloadState
|
||||
get() = context.image
|
||||
private var stopped: Boolean
|
||||
get() = context.stopped
|
||||
set(value) {
|
||||
context.stopped = value
|
||||
}
|
||||
private val httpContext: HttpClientContext
|
||||
get() = context.httpContext
|
||||
|
||||
init {
|
||||
httpContext.cookieStore = BasicCookieStore()
|
||||
httpContext.setAttribute(
|
||||
HTTPService.ContextAttributes.CONTEXT_ATTRIBUTES, HTTPService.ContextAttributes()
|
||||
)
|
||||
}
|
||||
|
||||
@Throws(DownloadException::class)
|
||||
fun download() {
|
||||
try {
|
||||
val image = image
|
||||
with(image) {
|
||||
this.status = Status.DOWNLOADING
|
||||
this.current = 0
|
||||
}
|
||||
dataTransaction.update(image)
|
||||
synchronized(image.postId.intern()) {
|
||||
val post = context.post
|
||||
if (post.status != Status.DOWNLOADING) {
|
||||
post.status = Status.DOWNLOADING
|
||||
dataTransaction.update(post)
|
||||
}
|
||||
}
|
||||
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
log.debug("Getting image url and name from ${image.url} using ${image.host}")
|
||||
val downloadedImage = image.host.downloadInternal(image.url, context)
|
||||
log.debug("Resolved name for ${image.url}: ${downloadedImage.name}")
|
||||
log.debug(
|
||||
"Downloaded image ${image.url} to ${downloadedImage.path}"
|
||||
)
|
||||
val sanitizedFileName = pathService.sanitize(downloadedImage.name)
|
||||
log.debug(
|
||||
"Sanitizing image name from ${downloadedImage.name} to $sanitizedFileName"
|
||||
)
|
||||
checkImageTypeAndRename(
|
||||
context.post, downloadedImage, image.index
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
throw DownloadException(e)
|
||||
} finally {
|
||||
synchronized(image.postId.intern()) {
|
||||
val image = image
|
||||
if (image.current == image.total && image.total > 0) {
|
||||
image.status = Status.FINISHED
|
||||
val post = context.post
|
||||
post.done += 1
|
||||
dataTransaction.update(post)
|
||||
} else if (stopped) {
|
||||
image.status = Status.STOPPED
|
||||
} else {
|
||||
image.status = Status.ERROR
|
||||
}
|
||||
dataTransaction.update(image)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(HostException::class)
|
||||
private fun checkImageTypeAndRename(
|
||||
postDownloadState: PostDownloadState, downloadedImage: DownloadedImage, index: Int
|
||||
) {
|
||||
val existingExtension = getExtension(downloadedImage.name)
|
||||
val extension = when (downloadedImage.type) {
|
||||
ImageMimeType.IMAGE_BMP -> "BMP"
|
||||
ImageMimeType.IMAGE_GIF -> "GIF"
|
||||
ImageMimeType.IMAGE_JPEG -> "JPG"
|
||||
ImageMimeType.IMAGE_PNG -> "PNG"
|
||||
ImageMimeType.IMAGE_WEBP -> "WEBP"
|
||||
}
|
||||
val filename = if (existingExtension.isBlank()) "${downloadedImage.name}.$extension" else downloadedImage.name
|
||||
try {
|
||||
val downloadDestinationFolder = Path.of(postDownloadState.downloadDirectory)
|
||||
synchronized(downloadDestinationFolder.pathString.intern()) {
|
||||
Files.createDirectories(downloadDestinationFolder)
|
||||
}
|
||||
val image = downloadDestinationFolder.resolve(
|
||||
"${
|
||||
if (settings.downloadSettings.forceOrder) String.format(
|
||||
"%03d_", index + 1
|
||||
) else ""
|
||||
}$filename"
|
||||
)
|
||||
Files.copy(downloadedImage.path, image, StandardCopyOption.REPLACE_EXISTING)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Failed to rename the image", e)
|
||||
} finally {
|
||||
try {
|
||||
Files.delete(downloadedImage.path)
|
||||
} catch (e: IOException) {
|
||||
log.warn(
|
||||
"Failed to delete temporary file ${downloadedImage.path}"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(Exception::class)
|
||||
override fun run() {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
download()
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (other == null || javaClass != other.javaClass) return false
|
||||
val that = other as ImageDownloadRunnable
|
||||
return imageInternalId == that.imageInternalId
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return Objects.hash(imageInternalId)
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
stopped = true
|
||||
val contextAttributes = httpContext.getAttribute(
|
||||
HTTPService.ContextAttributes.CONTEXT_ATTRIBUTES, HTTPService.ContextAttributes::class.java
|
||||
)
|
||||
if (contextAttributes != null) {
|
||||
synchronized(contextAttributes.requests) {
|
||||
for (request in contextAttributes.requests) {
|
||||
request.abort()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package me.mnlr.vripper.download
|
||||
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.function.CheckedSupplier
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.client.utils.URIBuilder
|
||||
import org.apache.http.util.EntityUtils
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.entities.LogEvent
|
||||
import me.mnlr.vripper.entities.LogEvent.Status.*
|
||||
import me.mnlr.vripper.exception.DownloadException
|
||||
import me.mnlr.vripper.exception.PostParseException
|
||||
import me.mnlr.vripper.formatToString
|
||||
import me.mnlr.vripper.model.PostItem
|
||||
import me.mnlr.vripper.parser.ThreadLookupAPIResponseHandler
|
||||
import me.mnlr.vripper.repositories.LogEventRepository
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.net.URISyntaxException
|
||||
import java.util.*
|
||||
import javax.xml.parsers.SAXParserFactory
|
||||
|
||||
class PostDownloadRunnable(private val threadId: String, private val postId: String) : Runnable {
|
||||
private val log by LoggerDelegate()
|
||||
private val dataTransaction: DataTransaction =
|
||||
SpringContext.getBean(DataTransaction::class.java)
|
||||
private val settingsService: SettingsService =
|
||||
SpringContext.getBean(SettingsService::class.java)
|
||||
private val vgAuthService: VGAuthService = SpringContext.getBean(VGAuthService::class.java)
|
||||
private val eventRepository: LogEventRepository =
|
||||
SpringContext.getBean(LogEventRepository::class.java)
|
||||
private val cm: HTTPService = SpringContext.getBean(HTTPService::class.java)
|
||||
private val retryPolicyService: RetryPolicyService =
|
||||
SpringContext.getBean(RetryPolicyService::class.java)
|
||||
private val downloadService: DownloadService =
|
||||
SpringContext.getBean(DownloadService::class.java)
|
||||
private val link: String =
|
||||
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId"
|
||||
private val logEvent: LogEvent
|
||||
|
||||
init {
|
||||
logEvent = eventRepository.save(
|
||||
LogEvent(
|
||||
type = LogEvent.Type.POST,
|
||||
status = PENDING,
|
||||
message = "Processing $link"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
try {
|
||||
eventRepository.update(logEvent.copy(status = PROCESSING))
|
||||
if (dataTransaction.exists(postId)) {
|
||||
log.warn(String.format("skipping %s, already loaded", postId))
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR,
|
||||
message = String.format("Gallery %s is already loaded", link)
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
val postItem: PostItem = try {
|
||||
parse()
|
||||
} catch (e: PostParseException) {
|
||||
val error = String.format("parsing failed for gallery %s", link)
|
||||
log.error(error, e)
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR, message = """
|
||||
$error
|
||||
${e.formatToString()}
|
||||
""".trimIndent()
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
if (postItem.imageItemList.isEmpty()) {
|
||||
val error = String.format("Gallery %s contains no images to download", link)
|
||||
log.error(error)
|
||||
eventRepository.update(logEvent.copy(status = ERROR, message = error))
|
||||
return
|
||||
}
|
||||
val post = dataTransaction.newPost(postItem)
|
||||
vgAuthService.leaveThanks(post)
|
||||
// metadataService.startFetchingMetadata(post)
|
||||
if (settingsService.settings.downloadSettings.autoStart) {
|
||||
log.debug("Auto start downloads option is enabled")
|
||||
downloadService.restartAll(listOf(postItem.postId))
|
||||
log.debug(String.format("Done enqueuing jobs for %s", postItem.url))
|
||||
}
|
||||
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = DONE,
|
||||
message = String.format(
|
||||
"Gallery %s is successfully added to download queue",
|
||||
link
|
||||
)
|
||||
)
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
val error = String.format("Error when adding gallery %s", link)
|
||||
log.error(error, e)
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR, message = """
|
||||
$error
|
||||
${e.formatToString()}
|
||||
""".trimIndent()
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(PostParseException::class)
|
||||
fun parse(): PostItem {
|
||||
log.debug("Parsing post $postId")
|
||||
val httpGet: HttpGet = try {
|
||||
val uriBuilder = URIBuilder("${settingsService.settings.viperSettings.host}/vr.php")
|
||||
uriBuilder.setParameter("p", postId)
|
||||
cm.buildHttpGet(uriBuilder.build(), HttpClientContext.create())
|
||||
} catch (e: URISyntaxException) {
|
||||
throw PostParseException(e)
|
||||
}
|
||||
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
|
||||
log.debug("Requesting $httpGet")
|
||||
return try {
|
||||
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
|
||||
log.error(
|
||||
"parsing failed for thread $threadId, post $postId", it.failure
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
val connection: HttpClient = cm.client.build()
|
||||
(connection.execute(
|
||||
httpGet, vgAuthService.context
|
||||
) as CloseableHttpResponse).use { response ->
|
||||
try {
|
||||
if (response.statusLine.statusCode / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.statusLine.statusCode}' for $httpGet")
|
||||
}
|
||||
factory.newSAXParser()
|
||||
.parse(response.entity.content, threadLookupAPIResponseHandler)
|
||||
threadLookupAPIResponseHandler.result.postItemList[0]
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (e: Exception) {
|
||||
throw PostParseException(e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val factory = SAXParserFactory.newInstance()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package me.mnlr.vripper.entities
|
||||
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import me.mnlr.vripper.host.Host
|
||||
|
||||
data class ImageDownloadState(
|
||||
var id: Long? = null,
|
||||
val postId: String,
|
||||
val url: String,
|
||||
val host: Host,
|
||||
val index: Int,
|
||||
val postIdRef: Long,
|
||||
var total: Long = -1,
|
||||
var current: Long = 0,
|
||||
var status: Status = Status.STOPPED,
|
||||
) {
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (javaClass != other?.javaClass) return false
|
||||
|
||||
other as ImageDownloadState
|
||||
|
||||
if (url != other.url) return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return url.hashCode()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package me.mnlr.vripper.entities
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat
|
||||
import java.time.LocalDateTime
|
||||
|
||||
data class LogEvent(
|
||||
val id: Long? = null,
|
||||
val type: Type,
|
||||
val status: Status,
|
||||
@JsonFormat(pattern="yyyy-MM-dd HH:mm:ss") val time: LocalDateTime = LocalDateTime.now(),
|
||||
val message: String,
|
||||
) {
|
||||
|
||||
enum class Type(val stringValue: String) {
|
||||
POST("Post"), THREAD("Thread"), THANKS("Thanks"), METADATA("Metadata"), SCAN("Scan"), DOWNLOAD(
|
||||
"Download"
|
||||
)
|
||||
}
|
||||
|
||||
enum class Status(val stringValue: String) {
|
||||
PENDING("Pending"), PROCESSING("Processing"), DONE("Done"), ERROR("Error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package me.mnlr.vripper.entities
|
||||
|
||||
class Metadata {
|
||||
var postIdRef: Long? = null
|
||||
var postId: String? = null
|
||||
var postedBy: String? = null
|
||||
var resolvedNames = emptyList<String>()
|
||||
|
||||
companion object {
|
||||
fun from(metadata: Metadata): Metadata {
|
||||
val copy = Metadata()
|
||||
copy.postIdRef = metadata.postIdRef
|
||||
copy.postId = metadata.postId
|
||||
copy.postedBy = metadata.postedBy
|
||||
copy.resolvedNames = metadata.resolvedNames
|
||||
return copy
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package me.mnlr.vripper.entities
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import java.time.LocalDateTime
|
||||
|
||||
data class PostDownloadState(
|
||||
var id: Long? = null,
|
||||
val postTitle: String,
|
||||
val threadTitle: String,
|
||||
val forum: String,
|
||||
val url: String,
|
||||
val token: String,
|
||||
val postId: String,
|
||||
val threadId: String,
|
||||
val total: Int,
|
||||
val hosts: Set<String>,
|
||||
val downloadDirectory: String,
|
||||
@JsonFormat(pattern="yyyy-MM-dd HH:mm:ss") val addedOn: LocalDateTime = LocalDateTime.now(),
|
||||
var status: Status = Status.STOPPED,
|
||||
var done: Int = 0,
|
||||
var rank: Int = Int.MAX_VALUE
|
||||
) {
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (javaClass != other?.javaClass) return false
|
||||
|
||||
other as PostDownloadState
|
||||
|
||||
if (postId != other.postId) return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return postId.hashCode()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package me.mnlr.vripper.entities
|
||||
|
||||
data class Thread(
|
||||
val id: Long? = null,
|
||||
val link: String,
|
||||
val threadId: String,
|
||||
var total: Int = 0,
|
||||
) {
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (javaClass != other?.javaClass) return false
|
||||
|
||||
other as Thread
|
||||
|
||||
return threadId == other.threadId
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return threadId.hashCode()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package me.mnlr.vripper.entities.domain
|
||||
|
||||
enum class Status(val stringValue: String) {
|
||||
PENDING("Pending"), DOWNLOADING("Downloading"), FINISHED("Finished"), ERROR("Error"), STOPPED("Stopped")
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package me.mnlr.vripper.event
|
||||
|
||||
data class Event<T>(val kind: Kind, val data: T) {
|
||||
enum class Kind {
|
||||
POST_UPDATE, POST_REMOVE, IMAGE_UPDATE, METADATA_UPDATE, THREAD_UPDATE, THREAD_REMOVE, LOG_EVENT_UPDATE, LOG_EVENT_REMOVE, VG_USER, DOWNLOAD_STATUS, BYTES_PER_SECOND, SETTINGS_UPDATE
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package me.mnlr.vripper.event
|
||||
|
||||
import jakarta.annotation.PreDestroy
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Flux
|
||||
import reactor.core.publisher.SignalType
|
||||
import reactor.core.publisher.Sinks
|
||||
import reactor.core.publisher.Sinks.EmitFailureHandler
|
||||
import reactor.core.publisher.Sinks.EmitResult
|
||||
|
||||
@Service
|
||||
class EventBus {
|
||||
private val sink = Sinks.many().multicast().onBackpressureBuffer<Event<*>>()
|
||||
fun publishEvent(event: Event<*>) {
|
||||
sink.emitNext(event, RETRY)
|
||||
}
|
||||
|
||||
fun flux(): Flux<Event<*>> {
|
||||
return sink.asFlux()
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private fun destroy() {
|
||||
sink.emitComplete(RETRY)
|
||||
}
|
||||
|
||||
companion object {
|
||||
val RETRY = EmitFailureHandler { _: SignalType?, _: EmitResult? -> true }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class DownloadException : Exception {
|
||||
constructor(message: String?) : super(message)
|
||||
constructor(e: Throwable?) : super(e)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class HostException : Exception {
|
||||
constructor(e: Throwable?) : super(e)
|
||||
constructor(message: String?) : super(message)
|
||||
constructor(message: String?, e: Throwable?) : super(message, e)
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class HtmlProcessorException(e: Throwable?) : Exception(e)
|
||||
@@ -0,0 +1,7 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class PostParseException : Exception {
|
||||
constructor(message: String?) : super(message)
|
||||
constructor(message: String?, e: Throwable?) : super(message, e)
|
||||
constructor(e: Throwable?) : super(e)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class QueueException : Exception {
|
||||
constructor(message: String?) : super(message)
|
||||
constructor(message: String?, e: Throwable?) : super(message, e)
|
||||
constructor(e: Throwable?) : super(e)
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class RenameException(message: String?, e: Exception?) : Exception(message, e)
|
||||
@@ -0,0 +1,3 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class ValidationException(message: String) : Exception(message)
|
||||
@@ -0,0 +1,6 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class VripperException : Exception {
|
||||
constructor(message: String?) : super(message)
|
||||
constructor(e: Throwable?) : super(e)
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package me.mnlr.vripper.exception
|
||||
|
||||
class XpathException(e: Throwable?) : Exception(e)
|
||||
@@ -0,0 +1,104 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.apache.http.NameValuePair
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class AcidimgHost(
|
||||
private val httpService: HTTPService,
|
||||
private val htmlProcessorService: HtmlProcessorService,
|
||||
private val xpathService: XpathService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
try {
|
||||
log.debug(
|
||||
String.format(
|
||||
"Looking for xpath expression %s in %s",
|
||||
CONTINUE_BUTTON_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
xpathService.getAsNode(document, CONTINUE_BUTTON_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
log.debug(String.format("Click button found for %s", url))
|
||||
val client: HttpClient = httpService.client.build()
|
||||
val httpPost = httpService.buildHttpPost(url, context.httpContext)
|
||||
httpPost.addHeader("Referer", url)
|
||||
val params: MutableList<NameValuePair> = ArrayList()
|
||||
params.add(BasicNameValuePair("imgContinue", "Continue to your image"))
|
||||
try {
|
||||
httpPost.entity = UrlEncodedFormEntity(params)
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
log.debug(String.format("Requesting %s", httpPost))
|
||||
val doc = try {
|
||||
(client.execute(
|
||||
httpPost, context.httpContext
|
||||
) as CloseableHttpResponse).use { response ->
|
||||
log.debug(String.format("Cleaning response for %s", httpPost))
|
||||
try {
|
||||
htmlProcessorService.clean(response.entity.content)
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim()
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim()
|
||||
Pair(
|
||||
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "acidimg.cc"
|
||||
private const val lookup = "acidimg.cc"
|
||||
private const val CONTINUE_BUTTON_XPATH = "//input[@id='continuebutton']"
|
||||
private const val IMG_XPATH = "//img[@class='centred']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class DPicMeHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { e: Node -> e.textContent.trim() }
|
||||
.orElse("")
|
||||
val imgUrl =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("src"))
|
||||
.map { e: Node -> e.textContent.trim() }
|
||||
.orElse("")
|
||||
var defaultName: String = UUID.randomUUID().toString()
|
||||
val index = imgUrl.lastIndexOf('/')
|
||||
if (index != -1 && index < imgUrl.length) {
|
||||
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1)
|
||||
}
|
||||
Pair((imgTitle.ifEmpty { defaultName })!!, imgUrl)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "dpic.me"
|
||||
private const val lookup = "dpic.me"
|
||||
private const val IMG_XPATH = "//img[@id='pic']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.apache.http.Header
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.util.EntityUtils
|
||||
import org.w3c.dom.Document
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.DownloadException
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.getFileNameWithoutExtension
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.io.FileOutputStream
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
abstract class Host(
|
||||
private val httpService: HTTPService,
|
||||
private val htmlProcessorService: HtmlProcessorService,
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val downloadSpeedService: DownloadSpeedService
|
||||
) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
companion object {
|
||||
private const val READ_BUFFER_SIZE = 8192
|
||||
}
|
||||
|
||||
abstract val host: String
|
||||
|
||||
@Throws(HostException::class)
|
||||
abstract fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String>
|
||||
|
||||
@Throws(HostException::class)
|
||||
fun downloadInternal(url: String, context: ImageDownloadContext): DownloadedImage {
|
||||
val headers = head(url, context.httpContext)
|
||||
// is the body of type image ?
|
||||
val imageMimeType = getImageMimeType(headers)
|
||||
|
||||
val downloadedImage = if (imageMimeType != null) {
|
||||
// a direct link, awesome
|
||||
val downloadedImage = fetch(url, context) {
|
||||
handleImageDownload(it, context)
|
||||
}
|
||||
DownloadedImage(getDefaultImageName(url), downloadedImage.first, downloadedImage.second)
|
||||
} else {
|
||||
// linked image ?
|
||||
val value = headers.find { it.name.contains("content-type", true) }?.value
|
||||
if (value != null) {
|
||||
if (value.contains("text/html")) {
|
||||
val document = fetch(url, context) {
|
||||
htmlProcessorService.clean(it.entity.content)
|
||||
}
|
||||
if (log.isDebugEnabled) {
|
||||
log.debug("Cleaning $url response", url)
|
||||
}
|
||||
val resolvedImage = resolve(url, document, context)
|
||||
val downloadImage: Pair<Path, ImageMimeType> =
|
||||
fetch(resolvedImage.second, context) {
|
||||
handleImageDownload(it, context)
|
||||
}
|
||||
DownloadedImage(resolvedImage.first, downloadImage.first, downloadImage.second)
|
||||
} else {
|
||||
throw HostException("Unable to download $url, can't process content type $value")
|
||||
}
|
||||
} else {
|
||||
throw HostException("Unexpected server response for $url, response have no content type")
|
||||
}
|
||||
}
|
||||
return downloadedImage
|
||||
}
|
||||
|
||||
private fun handleImageDownload(
|
||||
response: CloseableHttpResponse,
|
||||
context: ImageDownloadContext
|
||||
): Pair<Path, ImageMimeType> {
|
||||
val mimeType = getImageMimeType(response.allHeaders)
|
||||
?: throw HostException("Unsupported image type ${response.getFirstHeader("content-type")}")
|
||||
|
||||
val tempImage = Files.createTempFile("vripper", "tmp")
|
||||
return response.entity.content.use { inputStream ->
|
||||
try {
|
||||
FileOutputStream(tempImage.pathString).use { fos ->
|
||||
val image = context.image
|
||||
image.total = response.entity.contentLength
|
||||
dataTransaction.update(image)
|
||||
log.debug(
|
||||
"Length is ${image.total}"
|
||||
)
|
||||
log.debug(
|
||||
"Starting data transfer"
|
||||
)
|
||||
val buffer = ByteArray(READ_BUFFER_SIZE)
|
||||
var read: Int
|
||||
while (inputStream.read(buffer, 0, READ_BUFFER_SIZE)
|
||||
.also { read = it } != -1 && !context.stopped
|
||||
) {
|
||||
fos.write(buffer, 0, read)
|
||||
with(image) {
|
||||
current += read
|
||||
}
|
||||
downloadSpeedService.increase(read.toLong())
|
||||
dataTransaction.update(image)
|
||||
}
|
||||
Pair(tempImage, mimeType)
|
||||
}
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun isSupported(url: String): Boolean {
|
||||
return url.contains(host)
|
||||
}
|
||||
|
||||
@Throws(HostException::class)
|
||||
fun head(url: String, context: HttpClientContext): Array<Header> {
|
||||
val client: HttpClient = httpService.client.build()
|
||||
val httpGet = httpService.buildHttpHead(url, context)
|
||||
log.debug(String.format("Requesting %s", url))
|
||||
return try {
|
||||
(client.execute(
|
||||
httpGet,
|
||||
context
|
||||
) as CloseableHttpResponse).use { response ->
|
||||
try {
|
||||
if (response.statusLine.statusCode / 100 != 2) {
|
||||
throw HostException(
|
||||
String.format(
|
||||
"Unexpected response code: %d", response.statusLine.statusCode
|
||||
)
|
||||
)
|
||||
}
|
||||
response.allHeaders
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(HostException::class)
|
||||
fun <T> fetch(
|
||||
url: String,
|
||||
context: ImageDownloadContext,
|
||||
transformer: (CloseableHttpResponse) -> T
|
||||
): T {
|
||||
val client: HttpClient = httpService.client.build()
|
||||
val httpGet = httpService.buildHttpGet(url, context.httpContext)
|
||||
httpGet.addHeader("Referer", context.image.url)
|
||||
log.debug(String.format("Requesting %s", url))
|
||||
return try {
|
||||
(client.execute(
|
||||
httpGet,
|
||||
context.httpContext
|
||||
) as CloseableHttpResponse).use { response ->
|
||||
if (response.statusLine.statusCode / 100 != 2) {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
throw DownloadException(
|
||||
"Server returned code ${response.statusLine.statusCode}"
|
||||
)
|
||||
}
|
||||
try {
|
||||
transformer(response)
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getImageMimeType(headers: Array<Header>): ImageMimeType? {
|
||||
|
||||
// first check if content type header exists
|
||||
val value = headers.find { it.name.contains("content-type", true) }?.value
|
||||
|
||||
// header found, check the type
|
||||
return if (value != null) {
|
||||
ImageMimeType.values().find {
|
||||
value.contains(it.strValue, true)
|
||||
}
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
fun getDefaultImageName(imgUrl: String): String {
|
||||
val imageTitle = imgUrl.substring(imgUrl.lastIndexOf('/') + 1)
|
||||
log.debug(String.format("Extracting name from url %s: %s", imgUrl, imageTitle))
|
||||
return getFileNameWithoutExtension(imageTitle)
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (javaClass != other?.javaClass) return false
|
||||
|
||||
other as Host
|
||||
|
||||
if (host != other.host) return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return host.hashCode()
|
||||
}
|
||||
|
||||
override fun toString(): String {
|
||||
return host
|
||||
}
|
||||
}
|
||||
|
||||
data class DownloadedImage(val name: String, val path: Path, val type: ImageMimeType)
|
||||
|
||||
enum class ImageMimeType(val strValue: String) {
|
||||
IMAGE_BMP("image/bmp"),
|
||||
IMAGE_GIF("image/gif"),
|
||||
IMAGE_JPEG("image/jpeg"),
|
||||
IMAGE_PNG("image/png"),
|
||||
IMAGE_WEBP("image/webp"),
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.apache.http.impl.cookie.BasicClientCookie
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.sql.Date
|
||||
import java.time.LocalDateTime
|
||||
import java.time.ZoneId
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class ImageBamHost(
|
||||
private val htmlProcessorService: HtmlProcessorService,
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val doc = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, url))
|
||||
if (xpathService.getAsNode(document, CONTINUE_XPATH) != null) {
|
||||
val clientCookie = BasicClientCookie("nsfw_inter", "1")
|
||||
clientCookie.domain = "www.imagebam.com"
|
||||
clientCookie.path = "/"
|
||||
clientCookie.expiryDate =
|
||||
Date.from(
|
||||
LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant()
|
||||
)
|
||||
context.httpContext.cookieStore.addCookie(clientCookie)
|
||||
fetch(url, context) {
|
||||
htmlProcessorService.clean(it.entity.content)
|
||||
}
|
||||
} else {
|
||||
document
|
||||
}
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { e: Node -> e.textContent.trim { it <= ' ' } }
|
||||
.orElse("")
|
||||
val imgUrl = Optional.ofNullable(imgNode.attributes.getNamedItem("src"))
|
||||
.map { e: Node -> e.textContent.trim { it <= ' ' } }
|
||||
.orElse("")
|
||||
var defaultName: String = UUID.randomUUID().toString()
|
||||
val index = imgUrl.lastIndexOf('/')
|
||||
if (index != -1 && index < imgUrl.length) {
|
||||
defaultName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1)
|
||||
}
|
||||
Pair((imgTitle.ifEmpty { defaultName })!!, imgUrl)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imagebam.com"
|
||||
private const val lookup = "imagebam.com"
|
||||
private const val IMG_XPATH = "//img[contains(@class,'main-image')]"
|
||||
private const val CONTINUE_XPATH = "//*[contains(text(), 'Continue')]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class ImageTwistHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { obj: Node -> obj.textContent }
|
||||
.map { obj: String -> obj.trim { it <= ' ' } }.orElse(null)
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(imgTitle!!, imgUrl)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imagetwist.com"
|
||||
private const val lookup = "imagetwist.com"
|
||||
private const val IMG_XPATH = "//img[contains(@class, 'img')]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class ImageVenueHost(
|
||||
private val htmlProcessorService: HtmlProcessorService,
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val doc = try {
|
||||
log.debug(
|
||||
String.format(
|
||||
"Looking for xpath expression %s in %s",
|
||||
CONTINUE_BUTTON_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
if (xpathService.getAsNode(document, CONTINUE_BUTTON_XPATH) != null) {
|
||||
// Button detected. No need to actually click it, just make the call again.
|
||||
fetch(url, context) {
|
||||
htmlProcessorService.clean(it.entity.content)
|
||||
}
|
||||
} else {
|
||||
document
|
||||
}
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
if (imgTitle.isEmpty()) imgUrl.substring(imgUrl.lastIndexOf('/') + 1) else imgTitle,
|
||||
imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imagevenue.com"
|
||||
private const val lookup = "imagevenue.com"
|
||||
private const val CONTINUE_BUTTON_XPATH = "//a[@title='Continue to ImageVenue']"
|
||||
private const val IMG_XPATH = "//a[@data-toggle='full']/img"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class ImageZillaHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val titleNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
log.debug(String.format("Resolving name for %s", url))
|
||||
var title = titleNode.attributes.getNamedItem("title").textContent.trim()
|
||||
titleNode.textContent.trim()
|
||||
if (title.isEmpty()) {
|
||||
title = getDefaultImageName(url)
|
||||
}
|
||||
return try {
|
||||
Pair(title, url.replace("show", "images"))
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imagezilla.net"
|
||||
private const val lookup = "imagezilla.net/show"
|
||||
private const val IMG_XPATH = "//img[@id='photo']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class ImgSpiceHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
imgTitle.ifEmpty { imgUrl.substring(imgUrl.lastIndexOf('/') + 1) }, imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imgspice.com"
|
||||
private const val lookup = "imgspice.com"
|
||||
private const val IMG_XPATH = "//img[@id='imgpreview']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class ImgboxHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("title").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(imgTitle, imgUrl)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imgbox.com"
|
||||
private const val lookup = "imgbox.com"
|
||||
private const val IMG_XPATH = "//img[@id='img']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.apache.http.NameValuePair
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.HtmlProcessorException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.io.IOException
|
||||
|
||||
@Service
|
||||
class ImxHost(
|
||||
private val httpService: HTTPService,
|
||||
private val htmlProcessorService: HtmlProcessorService,
|
||||
private val xpathService: XpathService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
var value: String? = null
|
||||
try {
|
||||
log.debug("Looking for xpath expression $CONTINUE_BUTTON_XPATH in $url")
|
||||
val contDiv = xpathService.getAsNode(document, CONTINUE_BUTTON_XPATH)
|
||||
?: throw HostException("$CONTINUE_BUTTON_XPATH cannot be found")
|
||||
val node = contDiv.attributes.getNamedItem("value")
|
||||
if (node != null) {
|
||||
value = node.textContent
|
||||
}
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
if (value == null) {
|
||||
throw HostException("Failed to obtain value attribute from continue input")
|
||||
}
|
||||
log.debug("Click button found for $url")
|
||||
val client: HttpClient = httpService.client.build()
|
||||
val httpPost: HttpPost = httpService.buildHttpPost(url, context.httpContext)
|
||||
val params: MutableList<NameValuePair> = ArrayList()
|
||||
params.add(BasicNameValuePair("imgContinue", value))
|
||||
try {
|
||||
httpPost.entity = UrlEncodedFormEntity(params)
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
log.debug("Requesting $httpPost")
|
||||
val doc = try {
|
||||
(client.execute(
|
||||
httpPost, context.httpContext
|
||||
) as CloseableHttpResponse).use { response ->
|
||||
log.debug("Cleaning response for $httpPost")
|
||||
try {
|
||||
htmlProcessorService.clean(response.entity.content)
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
throw HostException(e)
|
||||
} catch (e: HtmlProcessorException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug("Looking for xpath expression $IMG_XPATH in $url")
|
||||
xpathService.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
"Xpath $IMG_XPATH cannot be found in $url"
|
||||
)
|
||||
return try {
|
||||
log.debug("Resolving name and image url for $url")
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "imx.to"
|
||||
private const val lookup = "imx.to"
|
||||
private const val CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']"
|
||||
private const val IMG_XPATH = "//img[@class='centred']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.net.URI
|
||||
import java.net.URISyntaxException
|
||||
|
||||
@Service
|
||||
class PimpandhostHost(
|
||||
private val xpathService: XpathService,
|
||||
private val htmlProcessorService: HtmlProcessorService,
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val newUrl: String
|
||||
try {
|
||||
newUrl = appendUri(url.replace("http://", "https://"), "size=original")
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
val doc = fetch(newUrl, context) {
|
||||
htmlProcessorService.clean(it.entity.content)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, newUrl))
|
||||
xpathService.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", newUrl))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl =
|
||||
"https:" + imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
imgTitle.ifEmpty { imgUrl.substring(imgUrl.lastIndexOf('/') + 1) }, imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(URISyntaxException::class)
|
||||
fun appendUri(uri: String, appendQuery: String): String {
|
||||
val oldUri = URI(uri)
|
||||
var newQuery = oldUri.query
|
||||
if (newQuery == null) {
|
||||
newQuery = appendQuery
|
||||
} else {
|
||||
newQuery += "&$appendQuery"
|
||||
}
|
||||
return URI(
|
||||
oldUri.scheme, oldUri.authority, oldUri.path, newQuery, oldUri.fragment
|
||||
).toString()
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "pimpandhost.com"
|
||||
private const val lookup = "pimpandhost.com"
|
||||
private const val IMG_XPATH = "//img[contains(@class, 'original')]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class PixRouteHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
Pair(imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' },
|
||||
imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "pixroute.com"
|
||||
private const val lookup = "pixroute.com"
|
||||
private const val IMG_XPATH = "//img[@id='imgpreview']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class PixhostHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "pixhost.to"
|
||||
private const val lookup = "pixhost.to"
|
||||
private const val IMG_XPATH = "//img[@id='image']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class PixxxelsHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
val titleNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
|
||||
xpathService.getAsNode(document, TITLE_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
TITLE_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = titleNode.textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
imgTitle.ifEmpty { imgUrl.substring(imgUrl.lastIndexOf('/') + 1) }, imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "pixxxels.cc"
|
||||
private const val lookup = "pixxxels.cc"
|
||||
private const val IMG_XPATH = "//*[@id='download']"
|
||||
private const val TITLE_XPATH = "//*[contains(@class,'imagename')]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class PostImgHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val titleNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
|
||||
xpathService.getAsNode(document, TITLE_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
TITLE_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
val urlNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle = Optional.ofNullable(titleNode)
|
||||
.map { node: Node -> node.textContent.trim { it <= ' ' } }
|
||||
.orElseGet { getDefaultImageName(url) }
|
||||
Pair(imgTitle, urlNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' })
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "postimg.cc"
|
||||
private const val lookup = "postimg.cc"
|
||||
private const val TITLE_XPATH = "//span[contains(@class,'imagename')]"
|
||||
private const val IMG_XPATH = "//a[@id='download']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
|
||||
@Service
|
||||
class TurboImageHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String
|
||||
get() = Companion.host
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
var title: String?
|
||||
title = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
|
||||
val titleNode: Node? = xpathService.getAsNode(document, TITLE_XPATH)
|
||||
log.debug(String.format("Resolving name for %s", url))
|
||||
titleNode?.textContent?.trim { it <= ' ' }
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
if (title.isNullOrEmpty()) {
|
||||
title = getDefaultImageName(url)
|
||||
}
|
||||
val urlNode: Node = xpathService.getAsNode(document, IMG_XPATH)
|
||||
?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return Pair(title, urlNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val host = "turboimagehost.com"
|
||||
private const val lookup = "turboimagehost.com"
|
||||
private const val TITLE_XPATH = "//div[contains(@class,'titleFullS')]/h1"
|
||||
private const val IMG_XPATH = "//img[@id='imageid']"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package me.mnlr.vripper.host
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.ImageDownloadContext
|
||||
import me.mnlr.vripper.exception.HostException
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import me.mnlr.vripper.services.*
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class ViprImHost(
|
||||
private val xpathService: XpathService,
|
||||
httpService: HTTPService,
|
||||
htmlProcessorService: HtmlProcessorService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host(httpService, htmlProcessorService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
override val host: String = "vipr.im"
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
xpathService.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
val imgTitle =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { obj: Node -> obj.textContent }
|
||||
.map { obj: String -> obj.trim() }.orElse(null)
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim()
|
||||
Pair(imgTitle!!, imgUrl)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val IMG_XPATH = "//img[contains(@class, 'img')]"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
data class DownloadSpeed(val speed: String) {
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (javaClass != other?.javaClass) return false
|
||||
|
||||
other as DownloadSpeed
|
||||
|
||||
if (speed != other.speed) return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return speed.hashCode()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
data class GlobalState(
|
||||
val running: Int,
|
||||
val remaining: Int,
|
||||
val error: Int,
|
||||
val loggedUser: String,
|
||||
val downloadSpeed: String
|
||||
) {
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (javaClass != other?.javaClass) return false
|
||||
|
||||
other as GlobalState
|
||||
|
||||
if (running != other.running) return false
|
||||
if (remaining != other.remaining) return false
|
||||
if (error != other.error) return false
|
||||
if (loggedUser != other.loggedUser) return false
|
||||
if (downloadSpeed != other.downloadSpeed) return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
var result = running
|
||||
result = 31 * result + remaining
|
||||
result = 31 * result + error
|
||||
result = 31 * result + loggedUser.hashCode()
|
||||
result = 31 * result + downloadSpeed.hashCode()
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
import me.mnlr.vripper.host.Host
|
||||
|
||||
data class ImageItem(val mainLink: String, val thumbLink: String, val host: Host)
|
||||
@@ -0,0 +1,3 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
data class LoggedUser(val user: String)
|
||||
@@ -0,0 +1,15 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
data class PostItem(
|
||||
val threadId: String,
|
||||
val threadTitle: String,
|
||||
val postId: String,
|
||||
val number: Int,
|
||||
val title: String,
|
||||
val imageCount: Int,
|
||||
val url: String,
|
||||
val hosts: List<Pair<String, Int>>,
|
||||
val securityToken: String,
|
||||
val forum: String,
|
||||
val imageItemList: List<ImageItem>
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
class Settings {
|
||||
var maxEventLog: Int = 1_000
|
||||
var connectionSettings = ConnectionSettings()
|
||||
var downloadSettings = DownloadSettings()
|
||||
var viperSettings = ViperSettings()
|
||||
var clipboardSettings = ClipboardSettings()
|
||||
}
|
||||
|
||||
data class ViperSettings(
|
||||
var login: Boolean = false,
|
||||
var username: String = "",
|
||||
var password: String = "",
|
||||
var thanks: Boolean = false,
|
||||
var host: String = "https://vipergirls.to",
|
||||
)
|
||||
|
||||
data class DownloadSettings(
|
||||
var downloadPath: String = System.getProperty("user.home"),
|
||||
var autoStart: Boolean = true,
|
||||
var autoQueueThreshold: Int = 1,
|
||||
var forceOrder: Boolean = false,
|
||||
var forumSubfolder: Boolean = false,
|
||||
var threadSubLocation: Boolean = false,
|
||||
var clearCompleted: Boolean = false,
|
||||
var appendPostId: Boolean = false
|
||||
)
|
||||
|
||||
data class ConnectionSettings(
|
||||
var maxThreads: Int = 2,
|
||||
var maxTotalThreads: Int = 0,
|
||||
var timeout: Int = 30,
|
||||
var maxAttempts: Int = 3,
|
||||
)
|
||||
|
||||
data class ClipboardSettings(var enable: Boolean = false, var pollingRate: Int = 500)
|
||||
@@ -0,0 +1,9 @@
|
||||
package me.mnlr.vripper.model
|
||||
|
||||
data class ThreadItem(
|
||||
val threadId: String,
|
||||
val title: String,
|
||||
val securityToken: String,
|
||||
val forum: String,
|
||||
val postItemList: List<PostItem>
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
package me.mnlr.vripper.parser
|
||||
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.function.CheckedSupplier
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.client.utils.URIBuilder
|
||||
import org.apache.http.util.EntityUtils
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.exception.DownloadException
|
||||
import me.mnlr.vripper.exception.PostParseException
|
||||
import me.mnlr.vripper.model.ThreadItem
|
||||
import me.mnlr.vripper.services.HTTPService
|
||||
import me.mnlr.vripper.services.RetryPolicyService
|
||||
import me.mnlr.vripper.services.SettingsService
|
||||
import me.mnlr.vripper.services.VGAuthService
|
||||
import java.io.BufferedInputStream
|
||||
import java.net.URISyntaxException
|
||||
import java.util.*
|
||||
import javax.xml.parsers.SAXParserFactory
|
||||
|
||||
class ThreadLookupAPIParser(private val threadId: String) {
|
||||
private val log by LoggerDelegate()
|
||||
private val cm: HTTPService = SpringContext.getBean(HTTPService::class.java)
|
||||
private val retryPolicyService: RetryPolicyService =
|
||||
SpringContext.getBean(RetryPolicyService::class.java)
|
||||
private val vgAuthService: VGAuthService = SpringContext.getBean(VGAuthService::class.java)
|
||||
private val settingsService: SettingsService =
|
||||
SpringContext.getBean(SettingsService::class.java)
|
||||
|
||||
@Throws(PostParseException::class)
|
||||
fun parse(): ThreadItem {
|
||||
log.debug("Parsing thread $threadId")
|
||||
val httpGet: HttpGet = try {
|
||||
val uriBuilder = URIBuilder(settingsService.settings.viperSettings.host + "/vr.php")
|
||||
uriBuilder.setParameter("t", threadId)
|
||||
cm.buildHttpGet(uriBuilder.build(), HttpClientContext.create())
|
||||
} catch (e: URISyntaxException) {
|
||||
throw PostParseException(e)
|
||||
}
|
||||
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
|
||||
log.debug("Requesting $httpGet")
|
||||
return try {
|
||||
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
|
||||
log.error(
|
||||
"parsing failed for thread $threadId",
|
||||
it.failure
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
val connection: HttpClient = cm.client.build()
|
||||
(connection.execute(
|
||||
httpGet, vgAuthService.context
|
||||
) as CloseableHttpResponse).use { response ->
|
||||
try {
|
||||
if (response.statusLine.statusCode / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.statusLine.statusCode}' for $httpGet")
|
||||
}
|
||||
factory.newSAXParser().parse(
|
||||
BufferedInputStream(response.entity.content),
|
||||
threadLookupAPIResponseHandler
|
||||
)
|
||||
threadLookupAPIResponseHandler.result
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (e: Exception) {
|
||||
throw PostParseException(e)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val factory = SAXParserFactory.newInstance()
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
package me.mnlr.vripper.parser
|
||||
|
||||
import org.xml.sax.Attributes
|
||||
import org.xml.sax.helpers.DefaultHandler
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.host.Host
|
||||
import me.mnlr.vripper.model.ImageItem
|
||||
import me.mnlr.vripper.model.PostItem
|
||||
import me.mnlr.vripper.model.ThreadItem
|
||||
import me.mnlr.vripper.services.SettingsService
|
||||
|
||||
class ThreadLookupAPIResponseHandler : DefaultHandler() {
|
||||
private val log by LoggerDelegate()
|
||||
private val supportedHosts: Collection<Host> =
|
||||
SpringContext.getBeansOfType(Host::class.java).values
|
||||
private val settingsService: SettingsService =
|
||||
SpringContext.getBean(SettingsService::class.java)
|
||||
private var error: String = ""
|
||||
private val hostMap: MutableMap<Host, Int> = mutableMapOf()
|
||||
private val postItemList: MutableList<PostItem> = mutableListOf()
|
||||
private val imageItemList: MutableList<ImageItem> = mutableListOf()
|
||||
private lateinit var threadItem: ThreadItem
|
||||
private lateinit var threadId: String
|
||||
private lateinit var threadTitle: String
|
||||
private lateinit var forum: String
|
||||
private lateinit var securityToken: String
|
||||
private lateinit var postId: String
|
||||
private lateinit var postTitle: String
|
||||
private var postCounter: Int = 0
|
||||
|
||||
val result: ThreadItem
|
||||
get() = threadItem
|
||||
|
||||
override fun startDocument() {}
|
||||
override fun startElement(
|
||||
uri: String, localName: String, qName: String, attributes: Attributes
|
||||
) {
|
||||
when (qName.lowercase()) {
|
||||
"error" -> error = attributes.getValue("details")
|
||||
"thread" -> {
|
||||
threadId = attributes.getValue("id")?.trim() ?: ""
|
||||
threadTitle = attributes.getValue("title")?.trim() ?: ""
|
||||
}
|
||||
|
||||
"forum" -> forum = attributes.getValue("title")?.trim() ?: ""
|
||||
"user" -> securityToken = attributes.getValue("hash")?.trim() ?: ""
|
||||
"post" -> {
|
||||
postId = attributes.getValue("id")?.trim() ?: ""
|
||||
postCounter = attributes.getValue("number")?.trim()?.toInt() ?: 0
|
||||
val title = attributes.getValue("title")?.trim() ?: ""
|
||||
postTitle = title.ifBlank { threadTitle }
|
||||
}
|
||||
|
||||
"image" -> {
|
||||
val mainLink = attributes.getValue("main_url")?.trim() ?: ""
|
||||
val thumbLink = attributes.getValue("thumb_url")?.trim() ?: ""
|
||||
val type = attributes.getValue("type")?.trim() ?: ""
|
||||
if(type == "linked") {
|
||||
try {
|
||||
val host = supportedHosts.first {
|
||||
it.isSupported(mainLink)
|
||||
}.let { host ->
|
||||
hostMap.computeIfAbsent(host) { 0 }
|
||||
hostMap[host] = hostMap[host]!! + 1
|
||||
host
|
||||
}
|
||||
imageItemList.add(ImageItem(mainLink, thumbLink, host))
|
||||
} catch (e: Exception) {
|
||||
log.warn("Unsupported link: $mainLink")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun endElement(uri: String, localName: String, qName: String) {
|
||||
if ("post".equals(qName, true)) {
|
||||
if (imageItemList.isNotEmpty()) {
|
||||
postItemList.add(
|
||||
PostItem(
|
||||
threadId,
|
||||
threadTitle,
|
||||
postId,
|
||||
postCounter,
|
||||
postTitle,
|
||||
imageItemList.size,
|
||||
"${settingsService.settings.viperSettings.host}/threads/?p=$postId&viewfull=1#post$postId",
|
||||
hostMap.toMap().map { Pair(it.key.host, it.value) },
|
||||
securityToken,
|
||||
forum,
|
||||
imageItemList.toList()
|
||||
)
|
||||
)
|
||||
}
|
||||
imageItemList.clear()
|
||||
hostMap.clear()
|
||||
}
|
||||
}
|
||||
|
||||
override fun endDocument() {
|
||||
threadItem = ThreadItem(threadId, threadTitle, securityToken, forum, postItemList.toList())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package me.mnlr.vripper.repositories
|
||||
|
||||
import me.mnlr.vripper.entities.ImageDownloadState
|
||||
import java.util.*
|
||||
|
||||
interface ImageRepository {
|
||||
fun save(imageDownloadState: ImageDownloadState): ImageDownloadState
|
||||
fun deleteAllByPostId(postId: String)
|
||||
fun findByPostId(postId: String): List<ImageDownloadState>
|
||||
fun countError(): Int
|
||||
fun findByPostIdAndIsNotCompleted(postId: String): List<ImageDownloadState>
|
||||
fun stopByPostIdAndIsNotCompleted(postId: String): Int
|
||||
fun findByPostIdAndIsError(postId: String): List<ImageDownloadState>
|
||||
fun findById(id: Long): Optional<ImageDownloadState>
|
||||
fun update(imageDownloadState: ImageDownloadState)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package me.mnlr.vripper.repositories
|
||||
|
||||
import me.mnlr.vripper.entities.LogEvent
|
||||
import java.util.*
|
||||
|
||||
interface LogEventRepository {
|
||||
fun save(logEvent: LogEvent): LogEvent
|
||||
fun update(logEvent: LogEvent): LogEvent
|
||||
fun findAll(): List<LogEvent>
|
||||
fun findById(id: Long): Optional<LogEvent>
|
||||
fun delete(id: Long)
|
||||
fun deleteAll()
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package me.mnlr.vripper.repositories
|
||||
|
||||
import me.mnlr.vripper.entities.Metadata
|
||||
import java.util.*
|
||||
|
||||
interface MetadataRepository {
|
||||
fun save(metadata: Metadata): Metadata
|
||||
fun findByPostId(postId: String): Optional<Metadata>
|
||||
fun deleteByPostId(postId: String): Int
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package me.mnlr.vripper.repositories
|
||||
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import java.util.*
|
||||
|
||||
interface PostDownloadStateRepository {
|
||||
fun save(postDownloadState: PostDownloadState): PostDownloadState
|
||||
fun findByPostId(postId: String): Optional<PostDownloadState>
|
||||
fun findById(id: Long): Optional<PostDownloadState>
|
||||
fun findCompleted(): List<String>
|
||||
fun findAll(): List<PostDownloadState>
|
||||
fun existByPostId(postId: String): Boolean
|
||||
fun setDownloadingToStopped(): Int
|
||||
fun deleteByPostId(postId: String): Int
|
||||
fun update(postDownloadState: PostDownloadState)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package me.mnlr.vripper.repositories
|
||||
|
||||
import me.mnlr.vripper.entities.Thread
|
||||
import java.util.*
|
||||
|
||||
interface ThreadRepository {
|
||||
fun save(thread: Thread): Thread
|
||||
fun findByThreadId(threadId: String): Optional<Thread>
|
||||
fun findAll(): List<Thread>
|
||||
fun findById(id: Long): Optional<Thread>
|
||||
fun deleteByThreadId(threadId: String): Int
|
||||
fun deleteAll()
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package me.mnlr.vripper.repositories.impl
|
||||
|
||||
import org.springframework.jdbc.core.JdbcTemplate
|
||||
import org.springframework.jdbc.core.RowMapper
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.entities.ImageDownloadState
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.host.Host
|
||||
import me.mnlr.vripper.repositories.ImageRepository
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class ImageRepositoryImpl(
|
||||
private val jdbcTemplate: JdbcTemplate, private val eventBus: EventBus
|
||||
) : ImageRepository {
|
||||
@Synchronized
|
||||
private fun nextId(): Long {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_IMAGE", Long::class.java)!!
|
||||
}
|
||||
|
||||
override fun save(imageDownloadState: ImageDownloadState): ImageDownloadState {
|
||||
val id = nextId()
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
imageDownloadState.current,
|
||||
imageDownloadState.host.host,
|
||||
imageDownloadState.index,
|
||||
imageDownloadState.postId,
|
||||
imageDownloadState.status.name,
|
||||
imageDownloadState.total,
|
||||
imageDownloadState.url,
|
||||
imageDownloadState.postIdRef
|
||||
)
|
||||
imageDownloadState.id = id
|
||||
eventBus.publishEvent(Event(Event.Kind.IMAGE_UPDATE, id))
|
||||
return imageDownloadState
|
||||
}
|
||||
|
||||
override fun deleteAllByPostId(postId: String) {
|
||||
jdbcTemplate.update("DELETE FROM IMAGE WHERE POST_ID = ?", postId)
|
||||
}
|
||||
|
||||
override fun findByPostId(postId: String): List<ImageDownloadState> {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE WHERE POST_ID = ?", ImageRowMapper(), postId
|
||||
)
|
||||
}
|
||||
|
||||
override fun countError(): Int {
|
||||
return jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM IMAGE AS image WHERE image.STATUS = ?", Int::class.java, Status.ERROR.name
|
||||
)
|
||||
}
|
||||
|
||||
override fun findByPostIdAndIsNotCompleted(postId: String): List<ImageDownloadState> {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS <> ?",
|
||||
ImageRowMapper(),
|
||||
postId,
|
||||
Status.FINISHED.name
|
||||
)
|
||||
}
|
||||
|
||||
override fun stopByPostIdAndIsNotCompleted(postId: String): Int {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = ? WHERE image.POST_ID = ? AND image.STATUS <> ?",
|
||||
Status.STOPPED.name,
|
||||
postId,
|
||||
Status.FINISHED.name
|
||||
)
|
||||
}
|
||||
|
||||
override fun findByPostIdAndIsError(postId: String): List<ImageDownloadState> {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.POST_ID = ? AND image.STATUS = ?",
|
||||
ImageRowMapper(),
|
||||
postId,
|
||||
Status.ERROR.name
|
||||
)
|
||||
}
|
||||
|
||||
override fun findById(id: Long): Optional<ImageDownloadState> {
|
||||
val images = jdbcTemplate.query(
|
||||
"SELECT * FROM IMAGE AS image WHERE image.ID = ?", ImageRowMapper(), id
|
||||
)
|
||||
return if (images.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(images[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun update(imageDownloadState: ImageDownloadState) {
|
||||
jdbcTemplate.update(
|
||||
"UPDATE IMAGE AS image SET image.STATUS = ?, image.CURRENT = ?, image.TOTAL = ? WHERE image.ID = ?",
|
||||
imageDownloadState.status.name,
|
||||
imageDownloadState.current,
|
||||
imageDownloadState.total,
|
||||
imageDownloadState.id
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.IMAGE_UPDATE, imageDownloadState.id))
|
||||
}
|
||||
}
|
||||
|
||||
internal class ImageRowMapper : RowMapper<ImageDownloadState> {
|
||||
@Throws(SQLException::class)
|
||||
override fun mapRow(rs: ResultSet, rowNum: Int): ImageDownloadState {
|
||||
val id = rs.getLong("ID")
|
||||
val postId = rs.getString("POST_ID")
|
||||
val url = rs.getString("URL")
|
||||
val host = SpringContext.getBeansOfType(Host::class.java).values.filter { it.host == rs.getString("HOST") }[0]
|
||||
val index = rs.getInt("INDEX")
|
||||
val current = rs.getLong("CURRENT")
|
||||
val total = rs.getLong("TOTAL")
|
||||
val status = Status.valueOf(rs.getString("STATUS"))
|
||||
val postIdRef = rs.getLong("POST_ID_REF")
|
||||
return ImageDownloadState(id, postId, url, host, index, postIdRef, total, current, status)
|
||||
}
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
package me.mnlr.vripper.repositories.impl
|
||||
|
||||
import org.springframework.jdbc.core.JdbcTemplate
|
||||
import org.springframework.jdbc.core.RowMapper
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.entities.LogEvent
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.repositories.LogEventRepository
|
||||
import me.mnlr.vripper.services.SettingsService
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.time.LocalDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class LogEventRepositoryImpl(
|
||||
private val jdbcTemplate: JdbcTemplate,
|
||||
private val settingsService: SettingsService,
|
||||
private val eventBus: EventBus
|
||||
) : LogEventRepository {
|
||||
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Synchronized
|
||||
private fun nextId(): Long {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_EVENT", Long::class.java)!!
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
override fun save(logEvent: LogEvent): LogEvent {
|
||||
val maxRecords = settingsService.settings.maxEventLog - 1
|
||||
val count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long::class.java) ?: 0
|
||||
if (count > maxRecords) {
|
||||
val idList = jdbcTemplate.queryForList(
|
||||
"SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?",
|
||||
Long::class.java,
|
||||
count - maxRecords
|
||||
)
|
||||
idList.forEach { delete(it) }
|
||||
}
|
||||
val id = nextId()
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
|
||||
id,
|
||||
logEvent.type.name,
|
||||
logEvent.status.name,
|
||||
logEvent.time.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
logEvent.message
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.LOG_EVENT_UPDATE, id))
|
||||
return logEvent.copy(id = id)
|
||||
}
|
||||
|
||||
override fun update(logEvent: LogEvent): LogEvent {
|
||||
if (logEvent.id == null) {
|
||||
log.warn("Cannot update entity with null id")
|
||||
return logEvent
|
||||
}
|
||||
jdbcTemplate.update(
|
||||
"UPDATE EVENT SET STATUS = ?, MESSAGE = ? WHERE ID = ?",
|
||||
logEvent.status.name,
|
||||
logEvent.message,
|
||||
logEvent.id
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.LOG_EVENT_UPDATE, logEvent.id))
|
||||
return logEvent
|
||||
}
|
||||
|
||||
override fun findById(id: Long): Optional<LogEvent> {
|
||||
val logEvents =
|
||||
jdbcTemplate.query("SELECT * FROM EVENT WHERE ID = ?", LogEventRowMapper(), id)
|
||||
return if (logEvents.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(logEvents[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun findAll(): List<LogEvent> {
|
||||
return jdbcTemplate.query("SELECT * FROM EVENT", LogEventRowMapper())
|
||||
}
|
||||
|
||||
override fun delete(id: Long) {
|
||||
jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id)
|
||||
eventBus.publishEvent(Event(Event.Kind.LOG_EVENT_REMOVE, id))
|
||||
}
|
||||
|
||||
override fun deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM EVENT")
|
||||
}
|
||||
}
|
||||
|
||||
internal class LogEventRowMapper : RowMapper<LogEvent> {
|
||||
@Throws(SQLException::class)
|
||||
override fun mapRow(rs: ResultSet, rowNum: Int): LogEvent {
|
||||
val id = rs.getLong("ID")
|
||||
val type = LogEvent.Type.valueOf(rs.getString("TYPE"))
|
||||
val status = LogEvent.Status.valueOf(rs.getString("STATUS"))
|
||||
val time =
|
||||
LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME)
|
||||
val message = rs.getString("MESSAGE")
|
||||
return LogEvent(id, type, status, time, message)
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package me.mnlr.vripper.repositories.impl
|
||||
|
||||
import org.springframework.jdbc.core.JdbcTemplate
|
||||
import org.springframework.jdbc.core.RowMapper
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.entities.Metadata
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.repositories.*
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class MetadataRepositoryImpl(
|
||||
private val jdbcTemplate: JdbcTemplate,
|
||||
private val eventBus: EventBus
|
||||
) : MetadataRepository {
|
||||
override fun save(metadata: Metadata): Metadata {
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
|
||||
metadata.postIdRef,
|
||||
metadata.postId,
|
||||
metadata.postedBy,
|
||||
java.lang.String.join("%sep%", metadata.resolvedNames)
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.METADATA_UPDATE, metadata.postIdRef))
|
||||
return metadata
|
||||
}
|
||||
|
||||
override fun findByPostId(postId: String): Optional<Metadata> {
|
||||
val metadata = jdbcTemplate.query(
|
||||
"SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
|
||||
MetadataRowMapper(),
|
||||
postId
|
||||
)
|
||||
return if (metadata.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(metadata[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun deleteByPostId(postId: String): Int {
|
||||
return jdbcTemplate.update(
|
||||
"DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?", postId
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
internal class MetadataRowMapper : RowMapper<Metadata> {
|
||||
@Throws(SQLException::class)
|
||||
override fun mapRow(rs: ResultSet, rowNum: Int): Metadata {
|
||||
val metadata = Metadata()
|
||||
metadata.postIdRef = rs.getLong("POST_ID_REF")
|
||||
metadata.postId = rs.getString("POST_ID")
|
||||
metadata.postedBy = rs.getString("POSTED_BY")
|
||||
val resolvedNames = rs.getString("RESOLVED_NAMES")
|
||||
if (resolvedNames != null && resolvedNames.isNotBlank()) {
|
||||
metadata.resolvedNames =
|
||||
resolvedNames.split("%sep%").dropLastWhile { it.isEmpty() }
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
}
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
package me.mnlr.vripper.repositories.impl
|
||||
|
||||
import org.springframework.jdbc.core.JdbcTemplate
|
||||
import org.springframework.jdbc.core.RowMapper
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.repositories.PostDownloadStateRepository
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.sql.Timestamp
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class PostDownloadStateRepositoryImpl(
|
||||
private val jdbcTemplate: JdbcTemplate, private val eventBus: EventBus
|
||||
) : PostDownloadStateRepository {
|
||||
@Synchronized
|
||||
private fun nextId(): Long {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_POST", Long::class.java)!!
|
||||
}
|
||||
|
||||
override fun save(postDownloadState: PostDownloadState): PostDownloadState {
|
||||
val id = nextId()
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO POST (ID, DONE, HOSTS, POST_FOLDER_NAME, POST_ID, STATUS, THREAD_ID, POST_TITLE, THREAD_TITLE, FORUM, TOTAL, URL, TOKEN, ADDED_ON, RANK) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id,
|
||||
postDownloadState.done,
|
||||
java.lang.String.join(";", postDownloadState.hosts),
|
||||
postDownloadState.downloadDirectory,
|
||||
postDownloadState.postId,
|
||||
postDownloadState.status.name,
|
||||
postDownloadState.threadId,
|
||||
postDownloadState.postTitle,
|
||||
postDownloadState.threadTitle,
|
||||
postDownloadState.forum,
|
||||
postDownloadState.total,
|
||||
postDownloadState.url,
|
||||
postDownloadState.token,
|
||||
Timestamp.valueOf(postDownloadState.addedOn),
|
||||
postDownloadState.rank
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.POST_UPDATE, id))
|
||||
return postDownloadState.copy(id = id)
|
||||
}
|
||||
|
||||
override fun findByPostId(postId: String): Optional<PostDownloadState> {
|
||||
val post = jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
|
||||
PostRowMapper(),
|
||||
postId
|
||||
)
|
||||
return if (post.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(post[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun findCompleted(): List<String> {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT POST_ID FROM POST AS post WHERE status = ? AND done >= total",
|
||||
{ rs: ResultSet, _: Int -> rs.getString("POST_ID") },
|
||||
Status.FINISHED.name
|
||||
)
|
||||
}
|
||||
|
||||
override fun findById(id: Long): Optional<PostDownloadState> {
|
||||
val post = jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.ID = ?",
|
||||
PostRowMapper(),
|
||||
id
|
||||
)
|
||||
return if (post.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(post[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun findAll(): List<PostDownloadState> {
|
||||
return jdbcTemplate.query(
|
||||
"SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID",
|
||||
PostRowMapper()
|
||||
)
|
||||
}
|
||||
|
||||
override fun existByPostId(postId: String): Boolean {
|
||||
val count = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM POST AS post WHERE post.POST_ID = ?", Int::class.java, postId
|
||||
)
|
||||
return count > 0
|
||||
}
|
||||
|
||||
override fun setDownloadingToStopped(): Int {
|
||||
return jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = ? WHERE post.STATUS = ? OR post.STATUS = ?",
|
||||
Status.STOPPED.name,
|
||||
Status.DOWNLOADING.name,
|
||||
Status.PENDING.name
|
||||
)
|
||||
}
|
||||
|
||||
override fun deleteByPostId(postId: String): Int {
|
||||
val mutationCount = jdbcTemplate.update("DELETE FROM POST AS post WHERE post.POST_ID = ?", postId)
|
||||
eventBus.publishEvent(Event(Event.Kind.POST_REMOVE, postId))
|
||||
return mutationCount
|
||||
}
|
||||
|
||||
override fun update(postDownloadState: PostDownloadState) {
|
||||
jdbcTemplate.update(
|
||||
"UPDATE POST AS post SET post.STATUS = ?, post.DONE = ?, post.RANK = ? WHERE post.ID = ?",
|
||||
postDownloadState.status.name,
|
||||
postDownloadState.done,
|
||||
postDownloadState.rank,
|
||||
postDownloadState.id
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.POST_UPDATE, postDownloadState.id))
|
||||
}
|
||||
}
|
||||
|
||||
internal class PostRowMapper : RowMapper<PostDownloadState> {
|
||||
@Throws(SQLException::class)
|
||||
override fun mapRow(rs: ResultSet, rowNum: Int): PostDownloadState {
|
||||
val id = rs.getLong("post.ID")
|
||||
val status = Status.valueOf(rs.getString("post.STATUS"))
|
||||
val postId = rs.getString("post.POST_ID")
|
||||
val threadId = rs.getString("post.THREAD_ID")
|
||||
val postTitle = rs.getString("post.POST_TITLE")
|
||||
val threadTitle = rs.getString("post.THREAD_TITLE")
|
||||
val forum = rs.getString("post.FORUM")
|
||||
val url = rs.getString("post.URL")
|
||||
val token = rs.getString("post.TOKEN")
|
||||
val done = rs.getInt("post.DONE")
|
||||
val total = rs.getInt("post.TOTAL")
|
||||
val hosts = rs.getString("post.HOSTS").split(DELIMITER).dropLastWhile { it.isEmpty() }.toSet()
|
||||
val downloadDirectory = rs.getString("post.POST_FOLDER_NAME")
|
||||
val addedOn = rs.getTimestamp("post.ADDED_ON").toLocalDateTime()
|
||||
val rank = rs.getInt("post.RANK")
|
||||
// val metadataId = rs.getLong("metadata.POST_ID_REF")
|
||||
// if (!rs.wasNull()) {
|
||||
// val metadata = Metadata()
|
||||
// metadata.postIdRef = metadataId
|
||||
// metadata.postId = rs.getString("metadata.POST_ID")
|
||||
// val resolvedNames = rs.getString("metadata.RESOLVED_NAMES")
|
||||
// if (resolvedNames != null && resolvedNames.isNotBlank()) {
|
||||
// metadata.resolvedNames =
|
||||
// resolvedNames.split("%sep%").dropLastWhile { it.isEmpty() }
|
||||
// }
|
||||
// metadata.postedBy = rs.getString("metadata.POSTED_BY")
|
||||
// postDownloadState.metadata = metadata
|
||||
// }
|
||||
return PostDownloadState(
|
||||
id, postTitle, threadTitle, forum, url, token, postId, threadId, total, hosts, downloadDirectory, addedOn, status, done, rank
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val DELIMITER = ";"
|
||||
}
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package me.mnlr.vripper.repositories.impl
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.jdbc.core.JdbcTemplate
|
||||
import org.springframework.jdbc.core.RowMapper
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.entities.Thread
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.repositories.ThreadRepository
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class ThreadRepositoryImpl @Autowired constructor(
|
||||
private val jdbcTemplate: JdbcTemplate,
|
||||
private val eventBus: EventBus
|
||||
) : ThreadRepository {
|
||||
@Synchronized
|
||||
private fun nextId(): Long {
|
||||
return jdbcTemplate.queryForObject("CALL NEXT VALUE FOR SEQ_THREAD", Long::class.java)!!
|
||||
}
|
||||
|
||||
override fun save(thread: Thread): Thread {
|
||||
val id = nextId()
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO THREAD (ID, TOTAL, LINK, THREAD_ID) values (?,?,?,?)",
|
||||
id,
|
||||
thread.total,
|
||||
thread.link,
|
||||
thread.threadId
|
||||
)
|
||||
eventBus.publishEvent(Event(Event.Kind.THREAD_UPDATE, id))
|
||||
return thread.copy(id = id)
|
||||
}
|
||||
|
||||
override fun findByThreadId(threadId: String): Optional<Thread> {
|
||||
val threadList = jdbcTemplate.query(
|
||||
"SELECT * FROM THREAD AS thread WHERE thread.THREAD_ID = ?",
|
||||
ThreadRowMapper(),
|
||||
threadId
|
||||
)
|
||||
return if (threadList.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(threadList[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun findAll(): List<Thread> {
|
||||
return jdbcTemplate.query("SELECT * FROM THREAD", ThreadRowMapper())
|
||||
}
|
||||
|
||||
override fun findById(id: Long): Optional<Thread> {
|
||||
val threadList = jdbcTemplate.query(
|
||||
"SELECT * FROM THREAD AS thread WHERE thread.ID = ?", ThreadRowMapper(), id
|
||||
)
|
||||
return if (threadList.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(threadList[0])
|
||||
}
|
||||
}
|
||||
|
||||
override fun deleteByThreadId(threadId: String): Int {
|
||||
val mutationCount =
|
||||
jdbcTemplate.update("DELETE FROM THREAD AS thread WHERE THREAD_ID = ?", threadId)
|
||||
eventBus.publishEvent(Event(Event.Kind.THREAD_REMOVE, threadId))
|
||||
return mutationCount
|
||||
}
|
||||
|
||||
override fun deleteAll() {
|
||||
jdbcTemplate.update("DELETE FROM THREAD")
|
||||
}
|
||||
}
|
||||
|
||||
internal class ThreadRowMapper : RowMapper<Thread> {
|
||||
@Throws(SQLException::class)
|
||||
override fun mapRow(rs: ResultSet, rowNum: Int): Thread {
|
||||
val id = rs.getLong("ID")
|
||||
val link = rs.getString("LINK")
|
||||
val threadId = rs.getString("THREAD_ID")
|
||||
val total = rs.getInt("TOTAL")
|
||||
return Thread(id, link, threadId, total)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import jakarta.annotation.PostConstruct
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.publisher.Sinks
|
||||
import reactor.core.scheduler.Schedulers
|
||||
|
||||
@Service
|
||||
class AsyncTaskRunnerService {
|
||||
val sink = Sinks.many().unicast().onBackpressureBuffer<Runnable>()
|
||||
|
||||
@PostConstruct
|
||||
fun init() {
|
||||
sink.asFlux().subscribeOn(Schedulers.parallel()).subscribe {
|
||||
it.run()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import me.mnlr.vripper.entities.ImageDownloadState
|
||||
import me.mnlr.vripper.entities.Metadata
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.entities.Thread
|
||||
import me.mnlr.vripper.entities.domain.Status
|
||||
import me.mnlr.vripper.model.PostItem
|
||||
import me.mnlr.vripper.repositories.ImageRepository
|
||||
import me.mnlr.vripper.repositories.MetadataRepository
|
||||
import me.mnlr.vripper.repositories.PostDownloadStateRepository
|
||||
import me.mnlr.vripper.repositories.ThreadRepository
|
||||
import org.springframework.stereotype.Service
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
@Service
|
||||
@Transactional
|
||||
class DataTransaction(
|
||||
private val postDownloadStateRepository: PostDownloadStateRepository,
|
||||
private val imageRepository: ImageRepository,
|
||||
private val threadRepository: ThreadRepository,
|
||||
private val metadataRepository: MetadataRepository,
|
||||
private val settingsService: SettingsService,
|
||||
private val pathService: PathService
|
||||
) {
|
||||
private fun save(postDownloadState: PostDownloadState): PostDownloadState {
|
||||
return postDownloadStateRepository.save(postDownloadState)
|
||||
}
|
||||
|
||||
fun update(postDownloadState: PostDownloadState) {
|
||||
postDownloadStateRepository.update(postDownloadState)
|
||||
}
|
||||
|
||||
fun save(thread: Thread): Thread {
|
||||
return threadRepository.save(thread)
|
||||
}
|
||||
|
||||
private fun save(imageDownloadState: ImageDownloadState): ImageDownloadState {
|
||||
return imageRepository.save(imageDownloadState)
|
||||
}
|
||||
|
||||
fun update(imageDownloadState: ImageDownloadState) {
|
||||
imageRepository.update(imageDownloadState)
|
||||
}
|
||||
|
||||
fun exists(postId: String): Boolean {
|
||||
return postDownloadStateRepository.existByPostId(postId)
|
||||
}
|
||||
|
||||
fun newPost(postItem: PostItem): PostDownloadState {
|
||||
val postDownloadState = save(
|
||||
PostDownloadState(
|
||||
postTitle = postItem.title,
|
||||
url = postItem.url,
|
||||
token = postItem.securityToken,
|
||||
postId = postItem.postId,
|
||||
threadId = postItem.threadId,
|
||||
total = postItem.imageCount,
|
||||
hosts = postItem.hosts.map { "${it.first} (${it.second})" }.toSet(),
|
||||
threadTitle = postItem.threadTitle,
|
||||
forum = postItem.forum,
|
||||
downloadDirectory = pathService.calculateDownloadPath(
|
||||
postItem.forum,
|
||||
postItem.threadTitle,
|
||||
postItem.title,
|
||||
postItem.postId,
|
||||
settingsService.settings
|
||||
).pathString
|
||||
)
|
||||
)
|
||||
val images: MutableList<ImageDownloadState> = mutableListOf()
|
||||
postItem.imageItemList.forEachIndexed { index, imageItem ->
|
||||
val imageDownloadState = ImageDownloadState(
|
||||
postId = postItem.postId,
|
||||
url = imageItem.mainLink,
|
||||
host = imageItem.host,
|
||||
index = index,
|
||||
postIdRef = postDownloadState.id!!
|
||||
)
|
||||
images.add(save(imageDownloadState))
|
||||
}
|
||||
sortPostsByRank()
|
||||
return postDownloadState
|
||||
}
|
||||
|
||||
fun finishPost(postDownloadState: PostDownloadState) {
|
||||
if (imageRepository.findByPostIdAndIsError(postDownloadState.postId).isNotEmpty()) {
|
||||
postDownloadState.status = Status.ERROR
|
||||
update(postDownloadState)
|
||||
} else {
|
||||
if (postDownloadState.done < postDownloadState.total) {
|
||||
postDownloadState.status = Status.STOPPED
|
||||
update(postDownloadState)
|
||||
} else {
|
||||
postDownloadState.status = Status.FINISHED
|
||||
update(postDownloadState)
|
||||
if (settingsService.settings.downloadSettings.clearCompleted) {
|
||||
remove(listOf(postDownloadState.postId))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun remove(postIds: List<String>) {
|
||||
for (postId in postIds) {
|
||||
imageRepository.deleteAllByPostId(postId)
|
||||
metadataRepository.deleteByPostId(postId)
|
||||
postDownloadStateRepository.deleteByPostId(postId)
|
||||
}
|
||||
sortPostsByRank()
|
||||
}
|
||||
|
||||
fun removeThread(threadId: String) {
|
||||
threadRepository.deleteByThreadId(threadId)
|
||||
}
|
||||
|
||||
fun clearCompleted(): List<String> {
|
||||
val completed = postDownloadStateRepository.findCompleted()
|
||||
remove(completed)
|
||||
return completed
|
||||
}
|
||||
|
||||
fun removeAll(postIds: List<String>?) {
|
||||
if (postIds != null) {
|
||||
remove(postIds)
|
||||
} else {
|
||||
remove(postDownloadStateRepository.findAll().map(PostDownloadState::postId))
|
||||
}
|
||||
}
|
||||
|
||||
fun stopImagesByPostIdAndIsNotCompleted(postId: String) {
|
||||
imageRepository.stopByPostIdAndIsNotCompleted(postId)
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun setMetadata(postDownloadState: PostDownloadState, metadata: Metadata) {
|
||||
if (metadataRepository.findByPostId(postDownloadState.postId).isEmpty) {
|
||||
metadata.postIdRef = postDownloadState.id
|
||||
metadataRepository.save(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
fun clearQueueLinks() {
|
||||
threadRepository.deleteAll()
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun sortPostsByRank() {
|
||||
val postDownloadState =
|
||||
postDownloadStateRepository.findAll().sortedWith(Comparator.comparing(PostDownloadState::addedOn))
|
||||
for (i in postDownloadState.indices) {
|
||||
postDownloadState[i].rank = i
|
||||
update(postDownloadState[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
@Service
|
||||
class DownloadSpeedService(private val eventBus: EventBus) {
|
||||
private val read = AtomicLong(0)
|
||||
var currentValue = 0L
|
||||
private var allowWrite = false
|
||||
fun increase(read: Long) {
|
||||
if (allowWrite) {
|
||||
this.read.addAndGet(read)
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 1000)
|
||||
private fun calc() {
|
||||
allowWrite = false
|
||||
val newValue = read.getAndSet(0)
|
||||
if (newValue != currentValue) {
|
||||
currentValue = newValue
|
||||
eventBus.publishEvent(Event(Event.Kind.BYTES_PER_SECOND, currentValue))
|
||||
}
|
||||
allowWrite = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.download.DownloadService
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.formatSI
|
||||
import me.mnlr.vripper.model.GlobalState
|
||||
import me.mnlr.vripper.repositories.ImageRepository
|
||||
|
||||
@Service
|
||||
class GlobalStateService(
|
||||
private val downloadService: DownloadService,
|
||||
private val imageRepository: ImageRepository,
|
||||
private val eventBus: EventBus,
|
||||
private val vgAuthService: VGAuthService,
|
||||
private val downloadSpeedService: DownloadSpeedService
|
||||
) {
|
||||
private var currentState: GlobalState = newValue()
|
||||
|
||||
@Scheduled(fixedDelay = 200)
|
||||
private fun interval() {
|
||||
val newGlobalState = newValue()
|
||||
if (newGlobalState != currentState) {
|
||||
currentState = newGlobalState
|
||||
eventBus.publishEvent(Event(Event.Kind.DOWNLOAD_STATUS, currentState))
|
||||
}
|
||||
}
|
||||
|
||||
private fun newValue(): GlobalState {
|
||||
return GlobalState(
|
||||
downloadService.runningCount(),
|
||||
downloadService.pendingCount(),
|
||||
imageRepository.countError(),
|
||||
vgAuthService.loggedUser,
|
||||
downloadSpeedService.currentValue.formatSI()
|
||||
)
|
||||
}
|
||||
|
||||
fun get(): GlobalState {
|
||||
return currentState.copy()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import jakarta.annotation.PreDestroy
|
||||
import org.apache.http.client.config.CookieSpecs
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.client.methods.AbstractExecutionAwareRequest
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.client.methods.HttpHead
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.impl.client.HttpClientBuilder
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.impl.client.LaxRedirectStrategy
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.Disposable
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.model.Settings
|
||||
import java.net.URI
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@Service
|
||||
class HTTPService(eventBus: EventBus, settingsService: SettingsService) {
|
||||
private val disposable: Disposable
|
||||
private lateinit var pcm: PoolingHttpClientConnectionManager
|
||||
private lateinit var rc: RequestConfig
|
||||
private var connectionTimeout: Int = settingsService.settings.connectionSettings.timeout
|
||||
|
||||
init {
|
||||
disposable = eventBus
|
||||
.flux()
|
||||
.filter { it.kind == Event.Kind.SETTINGS_UPDATE }
|
||||
.map { it.data as Settings }
|
||||
.subscribe {
|
||||
if (connectionTimeout != it.connectionSettings.timeout) {
|
||||
connectionTimeout = it.connectionSettings.timeout
|
||||
buildRequestConfig()
|
||||
}
|
||||
}
|
||||
buildRequestConfig()
|
||||
buildConnectionPool()
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private fun destroy() {
|
||||
disposable.dispose()
|
||||
}
|
||||
|
||||
private fun buildConnectionPool() {
|
||||
pcm = PoolingHttpClientConnectionManager()
|
||||
pcm.maxTotal = Int.MAX_VALUE
|
||||
pcm.defaultMaxPerRoute = Int.MAX_VALUE
|
||||
}
|
||||
|
||||
private fun buildRequestConfig() {
|
||||
rc = RequestConfig.custom()
|
||||
.setConnectionRequestTimeout(connectionTimeout * 1000)
|
||||
.setConnectTimeout(connectionTimeout * 1000)
|
||||
.setSocketTimeout(connectionTimeout * 1000)
|
||||
.setCookieSpec(CookieSpecs.STANDARD)
|
||||
.build()
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 15000)
|
||||
private fun idleConnectionMonitoring() {
|
||||
pcm.closeIdleConnections(60, TimeUnit.SECONDS)
|
||||
}
|
||||
|
||||
val client: HttpClientBuilder
|
||||
get() = HttpClients.custom()
|
||||
.setConnectionManager(pcm)
|
||||
.setRedirectStrategy(LaxRedirectStrategy())
|
||||
.disableAutomaticRetries()
|
||||
.setDefaultRequestConfig(rc)
|
||||
|
||||
fun buildHttpGet(url: String, context: HttpClientContext): HttpGet {
|
||||
val httpGet = HttpGet(url.replace(" ", "+"))
|
||||
httpGet.addHeader("User-Agent", USER_AGENT)
|
||||
addToContext(context, httpGet)
|
||||
return httpGet
|
||||
}
|
||||
|
||||
fun buildHttpHead(url: String, context: HttpClientContext): HttpHead {
|
||||
val httpHead = HttpHead(url.replace(" ", "+"))
|
||||
httpHead.addHeader("User-Agent", USER_AGENT)
|
||||
addToContext(context, httpHead)
|
||||
return httpHead
|
||||
}
|
||||
|
||||
fun buildHttpPost(url: String, context: HttpClientContext): HttpPost {
|
||||
val httpPost = HttpPost(url.replace(" ", "+"))
|
||||
httpPost.addHeader("User-Agent", USER_AGENT)
|
||||
addToContext(context, httpPost)
|
||||
return httpPost
|
||||
}
|
||||
|
||||
fun buildHttpGet(uri: URI, context: HttpClientContext): HttpGet {
|
||||
val httpGet = HttpGet(uri)
|
||||
httpGet.addHeader("User-Agent", USER_AGENT)
|
||||
addToContext(context, httpGet)
|
||||
return httpGet
|
||||
}
|
||||
|
||||
fun addToContext(context: HttpClientContext, request: AbstractExecutionAwareRequest) {
|
||||
val contextAttributes =
|
||||
context.getAttribute(
|
||||
ContextAttributes.CONTEXT_ATTRIBUTES,
|
||||
ContextAttributes::class.java
|
||||
)
|
||||
if (contextAttributes != null) {
|
||||
synchronized(contextAttributes.requests) {
|
||||
contextAttributes.requests.add(request)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ContextAttributes {
|
||||
val requests: MutableList<AbstractExecutionAwareRequest> = mutableListOf()
|
||||
|
||||
companion object {
|
||||
const val CONTEXT_ATTRIBUTES = "CONTEXT_ATTRIBUTES"
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import org.htmlcleaner.CleanerProperties
|
||||
import org.htmlcleaner.DomSerializer
|
||||
import org.htmlcleaner.HtmlCleaner
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Document
|
||||
import me.mnlr.vripper.exception.HtmlProcessorException
|
||||
import java.io.InputStream
|
||||
|
||||
@Service
|
||||
class HtmlProcessorService {
|
||||
@Throws(HtmlProcessorException::class)
|
||||
fun clean(htmlContent: InputStream): Document {
|
||||
return try {
|
||||
htmlContent.use {
|
||||
val clean = HtmlCleaner().clean(htmlContent)
|
||||
DomSerializer(CleanerProperties()).createDOM(clean)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
throw HtmlProcessorException(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.model.Settings
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
|
||||
@Service
|
||||
class PathService {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
fun calculateDownloadPath(forum: String, threadTitle: String, postTitle: String, postId: String, settings: Settings): Path {
|
||||
var downloadDirectory =
|
||||
if (settings.downloadSettings.forumSubfolder) Path.of(settings.downloadSettings.downloadPath, sanitize(forum)) else Path.of(
|
||||
settings.downloadSettings.downloadPath
|
||||
)
|
||||
downloadDirectory = if (settings.downloadSettings.threadSubLocation) downloadDirectory.resolve(threadTitle) else downloadDirectory
|
||||
downloadDirectory = downloadDirectory.resolve(if (settings.downloadSettings.appendPostId) "${sanitize(postTitle)}_${postId}" else sanitize(
|
||||
postTitle
|
||||
))
|
||||
return downloadDirectory
|
||||
}
|
||||
|
||||
private fun makeDir(sourceFolder: File): File {
|
||||
var counter = 1
|
||||
var folder = sourceFolder
|
||||
while (folder.exists()) {
|
||||
folder = File(sourceFolder.toString() + '.' + counter++)
|
||||
}
|
||||
if (!folder.mkdirs()) {
|
||||
throw RuntimeException(String.format("Failed to create the folder %s", sourceFolder))
|
||||
}
|
||||
return folder
|
||||
}
|
||||
|
||||
/**
|
||||
* Will sanitize the image name and remove extension
|
||||
*
|
||||
* @param path
|
||||
* @return Sanitized local path string
|
||||
*/
|
||||
fun sanitize(path: String): String {
|
||||
val sanitizedPath =
|
||||
path.replace("\\.|\\\\|/|\\||:|\\?|\\*|\"|<|>|\\p{Cntrl}".toRegex(), "_")
|
||||
log.debug(String.format("%s sanitized to %s", path, sanitizedPath))
|
||||
return sanitizedPath
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import jakarta.annotation.PreDestroy
|
||||
import net.jodah.failsafe.RetryPolicy
|
||||
import net.jodah.failsafe.event.ExecutionAttemptedEvent
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.Disposable
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.model.Settings
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
||||
@Service
|
||||
class RetryPolicyService(eventBus: EventBus, settingsService: SettingsService) {
|
||||
private val log by LoggerDelegate()
|
||||
private val disposable: Disposable
|
||||
private var maxAttempts: Int = settingsService.settings.connectionSettings.maxAttempts
|
||||
|
||||
init {
|
||||
disposable = eventBus
|
||||
.flux()
|
||||
.filter { it.kind == Event.Kind.SETTINGS_UPDATE }
|
||||
.map { it.data as Settings }
|
||||
.subscribe {
|
||||
if (maxAttempts != it.connectionSettings.maxAttempts) {
|
||||
maxAttempts = it.connectionSettings.maxAttempts
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> buildRetryPolicyForDownload(): RetryPolicy<T> {
|
||||
return RetryPolicy<T>()
|
||||
.withDelay(2, 5, ChronoUnit.SECONDS)
|
||||
.withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt {
|
||||
log.warn("#${it.attemptCount} tries failed", it.lastFailure)
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> buildGenericRetryPolicy(): RetryPolicy<T> {
|
||||
return RetryPolicy<T>()
|
||||
.withDelay(2, 5, ChronoUnit.SECONDS)
|
||||
.withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
|
||||
log.warn("#${e.attemptCount} tries failed", e.lastFailure)
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private fun destroy() {
|
||||
disposable.dispose()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
|
||||
import jakarta.annotation.PostConstruct
|
||||
import jakarta.annotation.PreDestroy
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.exception.ValidationException
|
||||
import me.mnlr.vripper.model.Settings
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.springframework.beans.factory.annotation.Value
|
||||
import org.springframework.core.io.Resource
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.FileWriter
|
||||
import java.io.IOException
|
||||
import java.nio.file.*
|
||||
|
||||
@Service
|
||||
class SettingsService(
|
||||
@Value("\${base.dir}") baseDir: String,
|
||||
@Value("\${base.dir.name}") baseDirName: String,
|
||||
private val eventBus: EventBus,
|
||||
@Value("classpath:proxies.json") private val defaultProxies: Resource
|
||||
) {
|
||||
private val log by LoggerDelegate()
|
||||
private val configPath: Path
|
||||
private val customProxiesPath: Path
|
||||
private val om = ObjectMapper(YAMLFactory())
|
||||
private val proxies: MutableSet<String> = HashSet()
|
||||
|
||||
|
||||
var settings = Settings()
|
||||
|
||||
init {
|
||||
configPath = Paths.get(baseDir, baseDirName, "config.yml")
|
||||
customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json")
|
||||
om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
||||
.findAndRegisterModules().registerKotlinModule()
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private fun init() {
|
||||
loadViperProxies()
|
||||
restore()
|
||||
eventBus.publishEvent(Event(Event.Kind.SETTINGS_UPDATE, settings))
|
||||
}
|
||||
|
||||
private fun loadViperProxies() {
|
||||
try {
|
||||
defaultProxies.inputStream.use { defaultProxiesIs ->
|
||||
val defaultProxies: List<String> = om.readValue(defaultProxiesIs)
|
||||
val customProxies: List<String> = if (customProxiesPath.toFile()
|
||||
.exists() && Files.isRegularFile(customProxiesPath)
|
||||
) {
|
||||
om.readValue(
|
||||
customProxiesPath.toFile()
|
||||
)
|
||||
} else {
|
||||
emptyList()
|
||||
}
|
||||
|
||||
try {
|
||||
FileWriter(customProxiesPath.toFile()).use { fw ->
|
||||
fw.append("[]").append(System.lineSeparator())
|
||||
fw.flush()
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
log.warn(
|
||||
"Unable to create " + customProxiesPath.toFile().absolutePath, e
|
||||
)
|
||||
}
|
||||
|
||||
proxies.addAll(customProxies)
|
||||
proxies.addAll(defaultProxies)
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
log.error("Failed to load vipergirls proxies list", e)
|
||||
}
|
||||
proxies.add("https://vipergirls.to")
|
||||
}
|
||||
|
||||
fun getProxies(): List<String> {
|
||||
return ArrayList(proxies)
|
||||
}
|
||||
|
||||
fun newSettings(settings: Settings) {
|
||||
if (settings.viperSettings.login) {
|
||||
if (this.settings.viperSettings.password != settings.viperSettings.password) {
|
||||
settings.viperSettings.password =
|
||||
DigestUtils.md5Hex(settings.viperSettings.password)
|
||||
}
|
||||
} else {
|
||||
settings.viperSettings.username = ""
|
||||
settings.viperSettings.password = ""
|
||||
settings.viperSettings.thanks = false
|
||||
settings.viperSettings.login = false
|
||||
}
|
||||
check(settings)
|
||||
this.settings = settings
|
||||
save()
|
||||
eventBus.publishEvent(Event(Event.Kind.SETTINGS_UPDATE, settings))
|
||||
}
|
||||
|
||||
fun restore() {
|
||||
try {
|
||||
if (configPath.toFile().exists()) {
|
||||
settings = om.readValue(configPath.toFile())
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
log.error("Failed restore user settings", e)
|
||||
settings = Settings()
|
||||
}
|
||||
if (!proxies.contains(settings.viperSettings.host)) {
|
||||
settings.viperSettings.host = "https://vipergirls.to"
|
||||
}
|
||||
try {
|
||||
check(settings)
|
||||
} catch (e: ValidationException) {
|
||||
log.error(
|
||||
String.format(
|
||||
"Your settings are invalid, either remove %s, or fix it", configPath
|
||||
), e
|
||||
)
|
||||
SpringContext.close()
|
||||
}
|
||||
save()
|
||||
}
|
||||
|
||||
fun save() {
|
||||
try {
|
||||
Files.write(
|
||||
configPath,
|
||||
om.writeValueAsBytes(settings),
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.WRITE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.SYNC
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
log.error("Failed to store user settings", e)
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private fun destroy() {
|
||||
save()
|
||||
}
|
||||
|
||||
@Throws(ValidationException::class)
|
||||
fun check(settings: Settings) {
|
||||
val path: Path = try {
|
||||
Paths.get(settings.downloadSettings.downloadPath)
|
||||
} catch (e: InvalidPathException) {
|
||||
throw ValidationException(
|
||||
String.format(
|
||||
"%s is invalid", settings.downloadSettings.downloadPath
|
||||
)
|
||||
)
|
||||
}
|
||||
if (!Files.exists(path)) {
|
||||
throw ValidationException(
|
||||
String.format(
|
||||
"%s does not exist", settings.downloadSettings.downloadPath
|
||||
)
|
||||
)
|
||||
} else if (!Files.isDirectory(path)) {
|
||||
throw ValidationException(
|
||||
String.format(
|
||||
"%s is not a directory", settings.downloadSettings.downloadPath
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
if (settings.downloadSettings.autoQueueThreshold < 0) {
|
||||
throw ValidationException("Invalid auto queue settings, value must be a positive integer")
|
||||
}
|
||||
|
||||
if (settings.connectionSettings.maxTotalThreads < 0 || settings.connectionSettings.maxTotalThreads > 12) {
|
||||
throw ValidationException(
|
||||
"Invalid max global concurrent download settings, values must be in [0,12]"
|
||||
)
|
||||
}
|
||||
if (settings.connectionSettings.maxThreads < 1 || settings.connectionSettings.maxThreads > 4) {
|
||||
throw ValidationException("Invalid max concurrent download settings, values must be in [1,4]")
|
||||
}
|
||||
if (settings.connectionSettings.timeout < 1 || settings.connectionSettings.timeout > 300) {
|
||||
throw ValidationException(
|
||||
"Invalid connection timeout settings, values must be in [1,300]"
|
||||
)
|
||||
}
|
||||
if (settings.connectionSettings.maxAttempts < 1 || settings.connectionSettings.maxAttempts > 10) {
|
||||
throw ValidationException("Invalid maximum attempts settings, values must be in [1,10]")
|
||||
}
|
||||
if (settings.maxEventLog < 100 || settings.maxEventLog > 10000) {
|
||||
throw ValidationException(
|
||||
"Invalid maximum event log record settings, values must be in [100,10000]"
|
||||
)
|
||||
}
|
||||
if (settings.clipboardSettings.pollingRate < 500) {
|
||||
throw ValidationException(
|
||||
"Invalid clipboard monitoring polling rate settings, values must be >= 500"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache
|
||||
import jakarta.annotation.PreDestroy
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.Disposable
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.model.ThreadItem
|
||||
import me.mnlr.vripper.parser.ThreadLookupAPIParser
|
||||
import java.util.*
|
||||
import java.util.concurrent.ExecutionException
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@Service
|
||||
class ThreadCacheService(eventBus: EventBus) {
|
||||
private val cache: LoadingCache<String, ThreadItem> =
|
||||
Caffeine.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES)
|
||||
.build { threadId: String ->
|
||||
ThreadLookupAPIParser(threadId).parse()
|
||||
}
|
||||
private val disposable: Disposable
|
||||
|
||||
init {
|
||||
disposable = eventBus.flux().filter { e: Event<*> -> e.kind == Event.Kind.SETTINGS_UPDATE }
|
||||
.subscribe { cache.invalidateAll() }
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private fun destroy() {
|
||||
disposable.dispose()
|
||||
}
|
||||
|
||||
@Throws(ExecutionException::class)
|
||||
operator fun get(threadId: String): ThreadItem {
|
||||
return cache[threadId] ?: throw NoSuchElementException("$threadId does not exist")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import jakarta.annotation.PostConstruct
|
||||
import jakarta.annotation.PreDestroy
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.event.Event
|
||||
import me.mnlr.vripper.event.EventBus
|
||||
import me.mnlr.vripper.exception.VripperException
|
||||
import me.mnlr.vripper.tasks.LeaveThanksRunnable
|
||||
import org.apache.http.NameValuePair
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.cookie.Cookie
|
||||
import org.apache.http.impl.client.BasicCookieStore
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
import org.springframework.stereotype.Service
|
||||
import reactor.core.Disposable
|
||||
|
||||
@Service
|
||||
class VGAuthService(
|
||||
private val cm: HTTPService,
|
||||
private val settingsService: SettingsService,
|
||||
private val asyncTaskRunnerService: AsyncTaskRunnerService,
|
||||
private val eventBus: EventBus
|
||||
) {
|
||||
private val log by LoggerDelegate()
|
||||
private val disposable: Disposable
|
||||
|
||||
final val context: HttpClientContext = HttpClientContext.create()
|
||||
var loggedUser = ""
|
||||
|
||||
var authenticated = false
|
||||
|
||||
init {
|
||||
context.cookieStore = BasicCookieStore()
|
||||
disposable = eventBus
|
||||
.flux()
|
||||
.filter { it.kind == Event.Kind.SETTINGS_UPDATE }
|
||||
.subscribe { authenticate() }
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private fun init() {
|
||||
authenticate()
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private fun destroy() {
|
||||
disposable.dispose()
|
||||
}
|
||||
|
||||
fun authenticate() {
|
||||
authenticated = false
|
||||
if (!settingsService.settings.viperSettings.login) {
|
||||
log.debug("Authentication option is disabled")
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(Event(Event.Kind.VG_USER, loggedUser))
|
||||
return
|
||||
}
|
||||
val username = settingsService.settings.viperSettings.username
|
||||
val password = settingsService.settings.viperSettings.password
|
||||
if (username.isEmpty() || password.isEmpty()) {
|
||||
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(Event(Event.Kind.VG_USER, loggedUser))
|
||||
return
|
||||
}
|
||||
val postAuth =
|
||||
cm.buildHttpPost(
|
||||
settingsService.settings.viperSettings.host + "/login.php?do=login",
|
||||
context
|
||||
)
|
||||
val params: MutableList<NameValuePair> = ArrayList()
|
||||
params.add(BasicNameValuePair("vb_login_username", username))
|
||||
params.add(BasicNameValuePair("cookieuser", "1"))
|
||||
params.add(BasicNameValuePair("do", "login"))
|
||||
params.add(BasicNameValuePair("vb_login_md5password", password))
|
||||
try {
|
||||
postAuth.entity = UrlEncodedFormEntity(params)
|
||||
} catch (e: Exception) {
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(Event(Event.Kind.VG_USER, loggedUser))
|
||||
log.error(
|
||||
"Failed to authenticate user with " + settingsService.settings.viperSettings.host,
|
||||
e
|
||||
)
|
||||
return
|
||||
}
|
||||
postAuth.addHeader("Referer", settingsService.settings.viperSettings.host)
|
||||
postAuth.addHeader(
|
||||
"Host",
|
||||
settingsService.settings.viperSettings.host.replace("https://", "")
|
||||
.replace("http://", "")
|
||||
)
|
||||
val client = cm.client.build()
|
||||
try {
|
||||
client.execute(postAuth, context).use { response ->
|
||||
if (response.statusLine.statusCode / 100 != 2) {
|
||||
throw VripperException(
|
||||
String.format(
|
||||
"Unexpected response code returned %s", response.statusLine.statusCode
|
||||
)
|
||||
)
|
||||
}
|
||||
val responseBody = EntityUtils.toString(response.entity)
|
||||
log.debug(
|
||||
String.format(
|
||||
"Authentication with ViperGirls response body:%n%s",
|
||||
responseBody
|
||||
)
|
||||
)
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
if (context.cookieStore.cookies.stream()
|
||||
.map { obj: Cookie -> obj.name }
|
||||
.noneMatch { e: String -> e == "vg_userid" }
|
||||
) {
|
||||
log.error(
|
||||
String.format(
|
||||
"Failed to authenticate user with %s, missing vg_userid cookie",
|
||||
settingsService.settings.viperSettings.host
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(Event(Event.Kind.VG_USER, loggedUser))
|
||||
log.error(
|
||||
"Failed to authenticate user with " + settingsService.settings.viperSettings.host,
|
||||
e
|
||||
)
|
||||
return
|
||||
}
|
||||
authenticated = true
|
||||
loggedUser = username
|
||||
log.info(String.format("Authenticated: %s", username))
|
||||
eventBus.publishEvent(Event(Event.Kind.VG_USER, loggedUser))
|
||||
}
|
||||
|
||||
fun leaveThanks(post: PostDownloadState) {
|
||||
asyncTaskRunnerService.sink
|
||||
.emitNext(LeaveThanksRunnable(post, authenticated, context)) { _, _ -> true }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package me.mnlr.vripper.services
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.w3c.dom.Node
|
||||
import org.w3c.dom.NodeList
|
||||
import me.mnlr.vripper.exception.XpathException
|
||||
import javax.xml.xpath.XPathConstants
|
||||
import javax.xml.xpath.XPathFactory
|
||||
|
||||
@Service
|
||||
class XpathService {
|
||||
private val xpath = XPathFactory.newInstance().newXPath()
|
||||
|
||||
@Throws(XpathException::class)
|
||||
fun getAsNode(source: Node?, xpathExpression: String?): Node? {
|
||||
return try {
|
||||
xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODE) as Node
|
||||
} catch (e: Exception) {
|
||||
throw XpathException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(XpathException::class)
|
||||
fun getAsNodeList(source: Node?, xpathExpression: String?): NodeList {
|
||||
return try {
|
||||
xpath.compile(xpathExpression).evaluate(source, XPathConstants.NODESET) as NodeList
|
||||
} catch (e: Exception) {
|
||||
throw XpathException(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package me.mnlr.vripper.tasks
|
||||
|
||||
import org.apache.http.NameValuePair
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.client.protocol.HttpClientContext
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.entities.LogEvent
|
||||
import me.mnlr.vripper.entities.LogEvent.Status.*
|
||||
import me.mnlr.vripper.entities.PostDownloadState
|
||||
import me.mnlr.vripper.formatToString
|
||||
import me.mnlr.vripper.repositories.LogEventRepository
|
||||
import me.mnlr.vripper.services.HTTPService
|
||||
import me.mnlr.vripper.services.SettingsService
|
||||
import java.io.UnsupportedEncodingException
|
||||
|
||||
class LeaveThanksRunnable(
|
||||
private val postDownloadState: PostDownloadState,
|
||||
private val authenticated: Boolean,
|
||||
private val context: HttpClientContext
|
||||
) : Runnable {
|
||||
private val log by LoggerDelegate()
|
||||
private val cm: HTTPService = SpringContext.getBean(HTTPService::class.java)
|
||||
private val settingsService: SettingsService = SpringContext.getBean(SettingsService::class.java)
|
||||
private val eventRepository: LogEventRepository = SpringContext.getBean(LogEventRepository::class.java)
|
||||
private val logEvent: LogEvent
|
||||
|
||||
init {
|
||||
logEvent = eventRepository.save(
|
||||
LogEvent(
|
||||
type = LogEvent.Type.THANKS, status = PENDING, message = "Leaving thanks for $postDownloadState.url"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
try {
|
||||
eventRepository.update(logEvent.copy(status = PROCESSING))
|
||||
if (!settingsService.settings.viperSettings.login) {
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = DONE,
|
||||
message = "Will not send a like for ${postDownloadState.url}\nAuthentication with ViperGirls option is disabled"
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
if (!settingsService.settings.viperSettings.thanks) {
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = DONE,
|
||||
message = "Will not send a like for ${postDownloadState.url}\nLeave thanks option is disabled"
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
if (!authenticated) {
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR,
|
||||
message = "Will not send a like for ${postDownloadState.url}\nYou are not authenticated"
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
val postThanks: HttpPost = cm.buildHttpPost(
|
||||
"${settingsService.settings.viperSettings.host}/post_thanks.php", HttpClientContext.create()
|
||||
)
|
||||
val params: MutableList<NameValuePair> = ArrayList()
|
||||
params.add(BasicNameValuePair("do", "post_thanks_add"))
|
||||
params.add(BasicNameValuePair("using_ajax", "1"))
|
||||
params.add(BasicNameValuePair("p", postDownloadState.postId))
|
||||
params.add(BasicNameValuePair("securitytoken", postDownloadState.token))
|
||||
try {
|
||||
postThanks.entity = UrlEncodedFormEntity(params)
|
||||
} catch (e: UnsupportedEncodingException) {
|
||||
val error = "Request error for ${postDownloadState.url}"
|
||||
log.error(error, e)
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR, message = """
|
||||
$error
|
||||
${e.formatToString()}
|
||||
""".trimIndent()
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
postThanks.addHeader("Referer", settingsService.settings.viperSettings.host)
|
||||
postThanks.addHeader(
|
||||
"Host", settingsService.settings.viperSettings.host.replace("https://", "").replace("http://", "")
|
||||
)
|
||||
val client: CloseableHttpClient = cm.client.build()
|
||||
client.execute(postThanks, context).use { response ->
|
||||
try {
|
||||
} finally {
|
||||
EntityUtils.consumeQuietly(response.entity)
|
||||
}
|
||||
}
|
||||
eventRepository.update(logEvent.copy(status = DONE))
|
||||
} catch (e: Exception) {
|
||||
val error = "Failed to leave a thanks for $postDownloadState"
|
||||
log.error(error, e)
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR, message = """
|
||||
$error
|
||||
${e.formatToString()}
|
||||
""".trimIndent()
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package me.mnlr.vripper.tasks
|
||||
|
||||
import me.mnlr.vripper.SpringContext
|
||||
import me.mnlr.vripper.delegate.LoggerDelegate
|
||||
import me.mnlr.vripper.download.PostDownloadRunnable
|
||||
import me.mnlr.vripper.entities.LogEvent
|
||||
import me.mnlr.vripper.entities.LogEvent.Status.*
|
||||
import me.mnlr.vripper.entities.Thread
|
||||
import me.mnlr.vripper.formatToString
|
||||
import me.mnlr.vripper.model.Settings
|
||||
import me.mnlr.vripper.model.ThreadItem
|
||||
import me.mnlr.vripper.repositories.LogEventRepository
|
||||
import me.mnlr.vripper.repositories.ThreadRepository
|
||||
import me.mnlr.vripper.services.AsyncTaskRunnerService
|
||||
import me.mnlr.vripper.services.DataTransaction
|
||||
import me.mnlr.vripper.services.SettingsService
|
||||
import me.mnlr.vripper.services.ThreadCacheService
|
||||
|
||||
class ThreadLookupRunnable(private val threadId: String, private val settings: Settings) :
|
||||
Runnable {
|
||||
private val log by LoggerDelegate()
|
||||
private val dataTransaction = SpringContext.getBean(DataTransaction::class.java)
|
||||
private val eventRepository = SpringContext.getBean(LogEventRepository::class.java)
|
||||
private val settingsService: SettingsService =
|
||||
SpringContext.getBean(SettingsService::class.java)
|
||||
private val threadCacheService = SpringContext.getBean(
|
||||
ThreadCacheService::class.java
|
||||
)
|
||||
private val threadRepository = SpringContext.getBean(
|
||||
ThreadRepository::class.java
|
||||
)
|
||||
private val asyncTaskRunnerService = SpringContext.getBean(
|
||||
AsyncTaskRunnerService::class.java
|
||||
)
|
||||
private val link: String =
|
||||
"${settingsService.settings.viperSettings.host}/threads/$threadId"
|
||||
private val logEvent: LogEvent
|
||||
|
||||
init {
|
||||
logEvent = eventRepository.save(
|
||||
LogEvent(
|
||||
type = LogEvent.Type.THREAD,
|
||||
status = LogEvent.Status.PENDING,
|
||||
message = "Processing multi-post link $link"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
try {
|
||||
eventRepository.update(logEvent.copy(status = PROCESSING))
|
||||
val threadLookupResult = threadCacheService[threadId]
|
||||
if (threadLookupResult.postItemList.isEmpty()) {
|
||||
val message = "Nothing found for $link"
|
||||
eventRepository.update(logEvent.copy(status = ERROR, message = message))
|
||||
return
|
||||
}
|
||||
if (threadRepository.findByThreadId(threadId).isEmpty) {
|
||||
dataTransaction.save(
|
||||
Thread(
|
||||
link = link,
|
||||
threadId = threadId,
|
||||
total = threadLookupResult.postItemList.size
|
||||
)
|
||||
)
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = DONE, message = "New thread $link is added"
|
||||
)
|
||||
)
|
||||
autostart(threadLookupResult)
|
||||
} else {
|
||||
log.info("Link $link is already loaded")
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR,
|
||||
message = "$link has already been added to the queue"
|
||||
)
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
val error = "Error when adding multi-post link $link"
|
||||
log.error(error, e)
|
||||
eventRepository.update(
|
||||
logEvent.copy(
|
||||
status = ERROR, message = """
|
||||
$error
|
||||
${e.formatToString()}
|
||||
""".trimIndent()
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun autostart(lookupResult: ThreadItem) {
|
||||
if (lookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
|
||||
lookupResult.postItemList.forEach {
|
||||
Pair(it.threadId, it.postId)
|
||||
asyncTaskRunnerService.sink.emitNext(
|
||||
PostDownloadRunnable(
|
||||
it.threadId, it.postId
|
||||
)
|
||||
) { _, _ -> true }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
logging.level.org.springframework.web=INFO
|
||||
logging.level.root=INFO
|
||||
logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats=ERROR
|
||||
logging.file.name=${base.dir}/${base.dir.name}/vripper.log
|
||||
server.port=${vripper.server.port:8080}
|
||||
management.endpoints.web.exposure.include=shutdown
|
||||
management.endpoint.shutdown.enabled=true
|
||||
server.error.include-message=always
|
||||
spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.xml
|
||||
spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/vripper;hsqldb.lock_file=false
|
||||
spring.datasource.username=SA
|
||||
spring.datasource.password=lEtmEIn
|
||||
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
|
||||
base.dir=${user.dir}
|
||||
base.dir.name=${vripper.base.dir.name:vripper}
|
||||
download.pool-size=12
|
||||
@@ -0,0 +1,145 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<databaseChangeLog xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
|
||||
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
|
||||
<changeSet author="vripper" id="0001">
|
||||
<createTable tableName="POST">
|
||||
<column name="ID" type="BIGINT">
|
||||
<constraints nullable="false" primaryKey="true" primaryKeyName="POST_PK"/>
|
||||
</column>
|
||||
<column name="DONE" type="INT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="HOSTS" type="VARCHAR(500)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="POST_FOLDER_NAME" type="VARCHAR(500)"/>
|
||||
<column name="POST_ID" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="STATUS" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="THREAD_ID" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="POST_TITLE" type="VARCHAR(500)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="THREAD_TITLE" type="VARCHAR(500)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="FORUM" type="VARCHAR(500)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="TOTAL" type="INT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="URL" type="VARCHAR(3000)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="TOKEN" type="VARCHAR(500)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column defaultValueComputed="CURRENT_TIMESTAMP" name="ADDED_ON" type="TIMESTAMP">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column defaultValue="0" name="RANK" type="INT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
</createTable>
|
||||
<createTable tableName="METADATA">
|
||||
<column name="POST_ID_REF" type="BIGINT">
|
||||
<constraints nullable="false" primaryKey="true" primaryKeyName="METADATA_PK" references="POST(ID)" foreignKeyName="METADATA_POST_ID_REF_POST_ID_FK"/>
|
||||
</column>
|
||||
<column name="POST_ID" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="POSTED_BY" type="VARCHAR(255)"/>
|
||||
<column name="RESOLVED_NAMES" type="VARCHAR(16777216)"/>
|
||||
</createTable>
|
||||
<createTable tableName="IMAGE">
|
||||
<column name="ID" type="BIGINT">
|
||||
<constraints nullable="false" primaryKey="true" primaryKeyName="IMAGE_PK"/>
|
||||
</column>
|
||||
<column name="CURRENT" type="BIGINT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="HOST" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="INDEX" type="INT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="POST_ID" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="STATUS" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="TOTAL" type="BIGINT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="URL" type="VARCHAR(3000)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="POST_ID_REF" type="BIGINT">
|
||||
<constraints nullable="false" references="POST(ID)" foreignKeyName="IMAGE_POST_ID_REF_POST_ID_FK"/>
|
||||
</column>
|
||||
</createTable>
|
||||
<createTable tableName="THREAD">
|
||||
<column name="ID" type="BIGINT">
|
||||
<constraints nullable="false" primaryKey="true" primaryKeyName="THREAD_PK"/>
|
||||
</column>
|
||||
<column name="TOTAL" type="INT">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="LINK" type="VARCHAR(3000)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="THREAD_ID" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
</createTable>
|
||||
<createTable tableName="EVENT">
|
||||
<column name="ID" type="BIGINT">
|
||||
<constraints nullable="false" primaryKey="true" primaryKeyName="EVENT_PK"/>
|
||||
</column>
|
||||
<column name="TYPE" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="STATUS" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="TIME" type="VARCHAR(255)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
<column name="MESSAGE" type="VARCHAR(16777216)">
|
||||
<constraints nullable="false"/>
|
||||
</column>
|
||||
</createTable>
|
||||
</changeSet>
|
||||
<changeSet author="vripper" id="0002">
|
||||
<createSequence incrementBy="1" sequenceName="SEQ_IMAGE" startValue="1"/>
|
||||
<createSequence incrementBy="1" sequenceName="SEQ_POST" startValue="1"/>
|
||||
<createSequence incrementBy="1" sequenceName="SEQ_THREAD" startValue="1"/>
|
||||
<createSequence incrementBy="1" sequenceName="SEQ_EVENT" startValue="1"/>
|
||||
</changeSet>
|
||||
<changeSet author="vripper" id="0003">
|
||||
<createIndex indexName="IMAGE_POST_ID_IDX" tableName="IMAGE">
|
||||
<column name="POST_ID"/>
|
||||
</createIndex>
|
||||
<createIndex indexName="IMAGE_STATUS_IDX" tableName="IMAGE">
|
||||
<column name="STATUS"/>
|
||||
</createIndex>
|
||||
<createIndex indexName="POST_POST_ID_IDX" tableName="POST">
|
||||
<column name="POST_ID"/>
|
||||
</createIndex>
|
||||
<createIndex indexName="POST_STATUS_IDX" tableName="POST">
|
||||
<column name="STATUS"/>
|
||||
</createIndex>
|
||||
<createIndex indexName="THREAD_THREAD_ID_IDX" tableName="THREAD">
|
||||
<column name="THREAD_ID"/>
|
||||
</createIndex>
|
||||
</changeSet>
|
||||
</databaseChangeLog>
|
||||
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<include resource=
|
||||
"org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
|
||||
<include resource=
|
||||
"org/springframework/boot/logging/logback/file-appender.xml"/>
|
||||
|
||||
<include resource=
|
||||
"org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
|
||||
<appender name="ASYNC-FILE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<appender-ref ref="FILE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-CONSOLE" class="ch.qos.logback.classic.AsyncAppender" >
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</appender>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="ASYNC-FILE"/>
|
||||
<appender-ref ref="ASYNC-CONSOLE"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -0,0 +1,8 @@
|
||||
[
|
||||
"https://planetviper.club",
|
||||
"https://viperbb.rocks",
|
||||
"https://viperkats.eu",
|
||||
"https://viperohilia.art",
|
||||
"https://viperproxy.org",
|
||||
"https://vipervault.link"
|
||||
]
|
||||
@@ -1,47 +0,0 @@
|
||||
# See http://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# compiled output
|
||||
/dist
|
||||
/tmp
|
||||
/out-tsc
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
|
||||
# profiling files
|
||||
chrome-profiler-events.json
|
||||
speed-measure-plugin.json
|
||||
|
||||
# IDEs and editors
|
||||
/.idea
|
||||
.project
|
||||
.classpath
|
||||
.c9/
|
||||
*.launch
|
||||
.settings/
|
||||
*.sublime-workspace
|
||||
|
||||
# IDE - VSCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
.history/*
|
||||
|
||||
# misc
|
||||
/.sass-cache
|
||||
/connect.lock
|
||||
/coverage
|
||||
/libpeerconnection.log
|
||||
npm-debug.log
|
||||
yarn-error.log
|
||||
testem.log
|
||||
/typings
|
||||
|
||||
# System Files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
dist
|
||||
build/vripper-ui
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"esversion": 6,
|
||||
"node": true,
|
||||
"globals": {
|
||||
"BigInt": false
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 28 KiB |
@@ -1,151 +0,0 @@
|
||||
const { app, BrowserWindow } = require("electron");
|
||||
const path = require("path");
|
||||
const url = require("url");
|
||||
const getPort = require("get-port");
|
||||
const { spawn } = require("child_process");
|
||||
const { ipcMain } = require("electron");
|
||||
const { dialog } = require("electron");
|
||||
const axios = require('axios');
|
||||
const appDir = process.env.APPDIR;
|
||||
|
||||
let win;
|
||||
let vripperServer;
|
||||
let serverPort;
|
||||
|
||||
const maxTerminationAttemps = 5;
|
||||
let terminationAttemps = 0;
|
||||
let terminationInteval;
|
||||
let terminated = false;
|
||||
|
||||
process.on("uncaughtException", err => {
|
||||
dialog.showErrorBox(err.message, err.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
function createWindow() {
|
||||
let icon;
|
||||
if(process.platform === "win32") {
|
||||
icon = __dirname + '/icon.ico';
|
||||
} else {
|
||||
icon = __dirname + '/icon.png';
|
||||
}
|
||||
win = new BrowserWindow({
|
||||
width: 1024,
|
||||
height: 800,
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
webPreferences: {
|
||||
nodeIntegration: true
|
||||
},
|
||||
icon: icon
|
||||
});
|
||||
|
||||
win.removeMenu();
|
||||
win.setMenu(null);
|
||||
win.setMenuBarVisibility(false);
|
||||
|
||||
win.loadURL(
|
||||
url.format({
|
||||
pathname: path.join(__dirname, `vripper-ui/index.html`),
|
||||
protocol: "file:",
|
||||
slashes: true
|
||||
})
|
||||
);
|
||||
|
||||
// win.webContents.openDevTools();
|
||||
|
||||
win.on("closed", () => {
|
||||
win = null;
|
||||
});
|
||||
}
|
||||
|
||||
getPort().then(port => {
|
||||
serverPort = port;
|
||||
ipcMain.on("get-port", event => {
|
||||
event.reply("port", port);
|
||||
});
|
||||
let javaBinPath;
|
||||
if(appDir !== undefined) {
|
||||
javaBinPath = path.join(appDir, "java-runtime/bin/java");
|
||||
} else {
|
||||
if(process.platform === 'darwin') {
|
||||
javaBinPath = path.join(app.getPath('exe'), "../../java-runtime/bin/java");
|
||||
} else {
|
||||
javaBinPath = path.join(app.getPath('exe'), "../java-runtime/bin/java");
|
||||
}
|
||||
}
|
||||
let jarPath;
|
||||
if(appDir !== undefined) {
|
||||
jarPath = path.join(appDir, "bin/vripper-server.jar");
|
||||
} else {
|
||||
if(process.platform === 'darwin') {
|
||||
jarPath = path.join(app.getPath('exe'), "../../bin/vripper-server.jar");
|
||||
} else {
|
||||
jarPath = path.join(app.getPath('exe'), "../bin/vripper-server.jar");
|
||||
}
|
||||
}
|
||||
vripperServer = spawn(javaBinPath, [
|
||||
"-Xms256m",
|
||||
"-Xmx1024m",
|
||||
"-Dvripper.server.port=" + port,
|
||||
"-jar",
|
||||
jarPath
|
||||
], {
|
||||
stdio: 'ignore'
|
||||
});
|
||||
vripperServer.on('exit', (code, signal) => {
|
||||
console.log(`vripper server terminated, code = ${code}, signal = ${signal}`);
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
});
|
||||
|
||||
const gotTheLock = app.requestSingleInstanceLock();
|
||||
|
||||
if (!gotTheLock) {
|
||||
app.quit();
|
||||
} else {
|
||||
app.on("second-instance", (event, commandLine, workingDirectory) => {
|
||||
if (win) {
|
||||
if (win.isMinimized()) win.restore();
|
||||
win.focus();
|
||||
}
|
||||
});
|
||||
|
||||
app.on("ready", createWindow);
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
if (process.platform !== "darwin") {
|
||||
axios.post('http://localhost:' + serverPort + '/actuator/shutdown', {}, {
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}).then((response) => {
|
||||
terminationInteval= setInterval(() => {
|
||||
terminationAttemps++;
|
||||
if(terminated) {
|
||||
console.log('viper server terminated');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
} else if(terminationAttemps > maxTerminationAttemps) {
|
||||
console.log('viper server is not terminated');
|
||||
console.log('Proceed to kill');
|
||||
vripperServer.kill('SIGKILL');
|
||||
clearInterval(terminationInteval);
|
||||
app.quit();
|
||||
}
|
||||
}, 1000);
|
||||
})
|
||||
.catch((error) => {
|
||||
// Terminate immediately
|
||||
vripperServer.kill('SIGKILL');
|
||||
terminated = true;
|
||||
app.quit();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
app.on("activate", () => {
|
||||
if (win === null) {
|
||||
createWindow();
|
||||
}
|
||||
});
|
||||
}
|
||||
Generated
-3082
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user