Compare commits

...
58 Commits
Author SHA1 Message Date
dev-clawandGitHub 0108b670c5 fixes #314 (#315) 2026-07-04 20:46:37 +01:00
dev-claw 378d493e2a Update version 2026-07-04 20:23:21 +01:00
dev-clawandGitHub 00468b7ed3 fixes #312 (#313) 2026-07-04 20:20:14 +01:00
dev-clawandGitHub 0978ff37b0 fixes #310 (#311) 2026-07-04 13:06:20 +01:00
dev-claw 5e52af0e00 Update version 2026-05-01 08:24:21 +01:00
dev-clawandGitHub 0dec4bd3f3 fixes #308 (#309) 2026-05-01 08:23:30 +01:00
dev-claw bb053ca314 Update README.md 2026-05-01 07:59:24 +01:00
dev-claw 681ee81d1e Update version 2026-04-30 23:09:11 +01:00
dev-claw bd027691b2 Change default for #303 2026-04-30 23:08:04 +01:00
dev-clawandGitHub 419f7e2ca7 Feature/issue 303/create host specific settings (#307)
* fixes #303

* fixes #303
2026-04-30 22:38:50 +01:00
dev-clawandGitHub 123ee14dd6 fixes #300 (#304) 2026-04-30 21:51:43 +01:00
dev-clawandGitHub a549105f7a fixes #302 (#305) 2026-04-30 21:51:33 +01:00
dev-clawandGitHub 8d1d74c5e8 fixes #303 (#306) 2026-04-30 21:51:24 +01:00
dev-claw eec3d5d75c Update version 2026-04-20 17:41:37 +01:00
dev-clawandGitHub 3f5127cbe3 fixes #298 (#299) 2026-04-20 17:40:32 +01:00
dev-claw b28fbb98c2 Update version 2026-04-18 22:00:37 +01:00
dev-clawandGitHub c806eb9d25 Merge pull request #297
* fixes #296
2026-04-18 21:59:57 +01:00
dev-claw 4ba422eee1 Update version 2026-04-18 01:39:04 +01:00
dev-claw e80d704b03 fixes #295
fixes #294
fixes #293
fixes #291
fixes #286
fixes #285
2026-04-18 01:37:49 +01:00
EVgZcQvc8iandGitHub 562c36bf28 Fix open file directory on Linux when the path contains spaces (#282) 2026-04-15 22:09:15 +01:00
dev-claw c3d07a8503 Update release.yml 2025-12-23 14:19:38 +01:00
dev-claw 30c9c9f24d Update version 2025-12-23 14:13:02 +01:00
dev-claw ecf74b7b00 bug fixes 2025-12-23 14:11:22 +01:00
dev-claw ed0c5f76a6 bug fixes 2025-10-12 14:08:00 +01:00
dev-claw 46ddb66dc7 bug fixes 2025-10-11 22:35:40 +01:00
dev-claw 11c9b665ce Update version 2025-10-11 17:39:25 +01:00
dev-claw 5b17f17e18 Update package.yml 2025-10-11 17:35:57 +01:00
dev-claw d84f19b4af Create package.yml 2025-10-11 17:20:49 +01:00
dev-claw 12fad45e8b remove release-manual.yml 2025-10-11 17:01:00 +01:00
dev-claw dc74e1573c bug fixes 2025-10-11 16:22:48 +01:00
dev-clawandGitHub f9b89c0b4d Update package.yml 2025-10-11 16:14:02 +01:00
dev-clawandGitHub 1bf75f0e8e Update release.yml 2025-10-10 21:51:44 +01:00
dev-clawandGitHub c13a9f93b3 Update release.yml (#278) 2025-10-10 21:49:32 +01:00
dev-claw 3302ceda20 Update release.yml & package.yml 2025-10-10 21:37:57 +01:00
dev-claw 3adf5a3254 Update version 2025-10-10 21:22:22 +01:00
dev-clawandGitHub ba22747384 fixes #274 (#277)
fixes #274
fixes #275
fixes #276
2025-10-10 21:06:04 +01:00
dev-claw 455833377d Update package.yml & release.yml 2025-10-10 21:02:32 +01:00
dev-claw 27be6e7167 Update package.yml 2025-10-10 20:41:45 +01:00
dev-clawandGitHub ab54aae48e Update README.md 2025-09-30 15:37:51 +01:00
dev-claw 34b31a51a5 version bump 2025-09-26 12:15:37 +01:00
dev-clawandGitHub 99a8aa0ce0 fixes #267 (#269) 2025-09-26 12:10:50 +01:00
dev-claw bdb67475a6 update README.md 2025-09-19 21:15:25 +01:00
dev-claw 24c961644e update README.md 2025-09-19 21:07:39 +01:00
dev-claw 3c3b79f581 update README.md 2025-09-19 20:22:09 +01:00
dev-claw 8c30e5a02d fixes #239 2025-09-19 20:09:05 +01:00
dev-claw 398220d518 update README.md 2025-09-19 20:04:54 +01:00
dev-clawandGitHub af9a439d81 fixes #239 (#266) 2025-09-19 19:13:18 +01:00
dev-claw c9760511b5 version bump 2025-09-19 19:03:27 +01:00
dev-clawandGitHub 59fe4d227f fixes #239 (#265) 2025-09-19 19:02:36 +01:00
dev-clawandGitHub 6df84c0756 fixes #261 (#264) 2025-09-19 18:41:04 +01:00
dev-clawandGitHub aa0119f881 fixes #239 (#263) 2025-09-19 18:29:03 +01:00
dev-clawandGitHub 9283ab7837 fixes #260 (#262) 2025-09-06 14:19:34 +01:00
dev-claw ca77074ef6 version bump 2025-08-19 15:42:07 +01:00
dev-clawandGitHub 3cb251cb36 fixes #250 (#252) 2025-08-19 15:38:08 +01:00
dev-clawandGitHub 73f478c1bc fixes #248 (#249) 2025-08-18 19:46:05 +01:00
dev-claw bb1dfbfee1 version bump 2025-08-18 19:36:51 +01:00
dev-clawandGitHub a89fe8571c fixes #243 (#246) 2025-08-18 19:34:21 +01:00
dev-clawandGitHub 44c8e443f4 fixes #240 (#241) 2025-08-08 15:41:43 +01:00
129 changed files with 3307 additions and 2294 deletions
+174 -37
View File
@@ -2,54 +2,191 @@ name: Package
on:
workflow_dispatch:
inputs:
release:
description: 'Release'
required: true
type: string
jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ]
runs-on: ${{ matrix.os }}
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
settings-path: ${{ github.workspace }} # location for the settings.xml file
- name: Set up jdk25
uses: actions/setup-java@v3
with:
java-version: '25'
distribution: 'zulu'
- name: Build with Maven
run: mvn -B install --file pom.xml
- name: Build gui jar
run: |
mvn -N -q install --file pom.xml
mvn -B -q install --file vripper-core/pom.xml
mvn -B -q install --file vripper-gui/pom.xml
mv vripper-gui/target/vripper-gui-${{ inputs.release }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ inputs.release }}.jar
- name: Rename GUI artifact
run: mv vripper-gui/target/vripper-gui-*.jar vripper-gui/target/vripper-gui.jar
- if: matrix.os == 'ubuntu-latest'
name: Release gui Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-gui/target/vripper-noarch-gui-${{ inputs.release }}.jar
# Start building WEB jar in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Build web Jar
run: |
mvn -B -q install --file vripper-web-ui/pom.xml
mvn -B -q install --file vripper-web/pom.xml
mv vripper-web/target/vripper-web-${{ inputs.release }}.jar vripper-web/target/vripper-noarch-web-${{ inputs.release }}.jar
- name: Rename WEB artifact
run: mv vripper-web/target/vripper-web-*.jar vripper-web/target/vripper-web.jar
- if: matrix.os == 'ubuntu-latest'
name: Release web Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-web/target/vripper-noarch-web-${{ inputs.release }}.jar
# End building WEB jar in ubuntu only
# Start building docker image in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- if: matrix.os == 'ubuntu-latest'
name: Package Linux binaries
run: |
cd jpackage
cp ../vripper-gui/target/vripper-gui.jar jar/vripper-gui.jar
jpackage --type deb "@jpackage.cfg" "@jpackage-linux.cfg"
jpackage --type rpm "@jpackage.cfg" "@jpackage-linux.cfg"
ls -la dist
- if: matrix.os == 'ubuntu-latest'
name: Build Docker image
run: |
mkdir docker_build && cp Dockerfile docker_build/Dockerfile && cp vripper-web/target/vripper-noarch-web-${{ inputs.release }}.jar docker_build/vripper-web.jar
docker build -t ghcr.io/dev-claw/vripper-web docker_build
docker tag ghcr.io/dev-claw/vripper-web:latest ghcr.io/dev-claw/vripper-web:${{ inputs.release }}
docker push ghcr.io/dev-claw/vripper-web:latest
docker push ghcr.io/dev-claw/vripper-web:${{ inputs.release }}
# End building docker image in ubuntu only
- if: matrix.os == 'windows-latest'
name: Package Windows binaries
run: |
cd jpackage
cp ../vripper-gui/target/vripper-gui.jar jar/vripper-gui.jar
jpackage "@jpackage.cfg" "@jpackage-windows.cfg"
dir dist
- name: Prepare Packaging
run: |
cp vripper-gui/target/vripper-noarch-gui-${{ inputs.release }}.jar jpackage/jar/vripper-gui.jar
- if: matrix.os == 'macos-latest'
name: Package macOS binaries
run: |
cd jpackage
cp ../vripper-gui/target/vripper-gui.jar jar/vripper-gui.jar
jpackage "@jpackage.cfg" "@jpackage-macos.cfg"
ls -la dist
- if: matrix.os == 'ubuntu-latest'
name: Package for Linux
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.png
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type deb
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type rpm
mv dist/vripper-${{ inputs.release }}-1.x86_64.rpm dist/vripper-linux-${{ inputs.release }}.x86_64.rpm
mv dist/vripper_${{ inputs.release }}-1_amd64.deb dist/vripper-linux-${{ inputs.release }}_amd64.deb
- if: matrix.os == 'windows-latest'
name: Package for Windows
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.ico
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type msi
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type exe
cd dist
ren VRipper-${{ inputs.release }}.msi vripper-windows-installer-${{ inputs.release }}.msi
ren VRipper-${{ inputs.release }}.exe vripper-windows-installer-${{ inputs.release }}.exe
- if: matrix.os == 'macos-latest'
name: Package for macOS(arm64)
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ inputs.release }}.pkg dist/vripper-macos-${{ inputs.release }}.arm64.pkg
mv dist/VRipper-${{ inputs.release }}.dmg dist/vripper-macos-${{ inputs.release }}.arm64.dmg
- if: matrix.os == 'macos-13'
name: Package for macOS(x86_64)
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ inputs.release }}.pkg dist/vripper-macos-${{ inputs.release }}.x86_64.pkg
mv dist/VRipper-${{ inputs.release }}.dmg dist/vripper-macos-${{ inputs.release }}.x86_64.dmg
- if: matrix.os == 'ubuntu-latest'
name: Zip Ubuntu portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper'
filename: 'vripper-linux-portable-${{ inputs.release }}.zip'
- if: matrix.os == 'windows-latest'
name: Zip Windows portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper'
filename: 'vripper-windows-portable-${{ inputs.release }}.zip'
- if: matrix.os == 'macos-latest'
name: Zip macOS(arm64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ inputs.release }}.arm64.zip'
- if: matrix.os == 'macos-13'
name: Zip macOS(x86_64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ inputs.release }}.x86_64.zip'
- if: matrix.os == 'ubuntu-latest'
name: Release packages for Linux
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-linux-${{ inputs.release }}.x86_64.rpm
jpackage/dist/vripper-linux-${{ inputs.release }}_amd64.deb
jpackage/dist/vripper-linux-portable-${{ inputs.release }}.zip
- if: matrix.os == 'windows-latest'
name: Release packages for Windows
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-windows-installer-${{ inputs.release }}.msi
jpackage/dist/vripper-windows-installer-${{ inputs.release }}.exe
jpackage/dist/vripper-windows-portable-${{ inputs.release }}.zip
- if: matrix.os == 'macos-latest'
name: Release packages for macOS(arm64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ inputs.release }}.arm64.pkg
jpackage/dist/vripper-macos-${{ inputs.release }}.arm64.dmg
jpackage/dist/vripper-macos-portable-${{ inputs.release }}.arm64.zip
- if: matrix.os == 'macos-13'
name: Release packages for macOS(x86_64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ inputs.release }}.x86_64.pkg
jpackage/dist/vripper-macos-${{ inputs.release }}.x86_64.dmg
jpackage/dist/vripper-macos-portable-${{ inputs.release }}.x86_64.zip
+45 -23
View File
@@ -8,18 +8,20 @@ jobs:
build:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ]
os: [ ubuntu-latest, windows-latest, macos-15-intel, macos-latest ]
runs-on: ${{ matrix.os }}
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v3
- name: Set up jdk21
- name: Set up jdk25
uses: actions/setup-java@v3
with:
java-version: '21'
distribution: 'temurin'
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
settings-path: ${{ github.workspace }} # location for the settings.xml file
java-version: '25'
distribution: 'zulu'
cache: maven
- name: Build gui jar
run: |
@@ -49,6 +51,26 @@ jobs:
files: vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
# End building WEB jar in ubuntu only
# Start building docker image in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- if: matrix.os == 'ubuntu-latest'
name: Build Docker image
run: |
mkdir docker_build && cp Dockerfile docker_build/Dockerfile && cp vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar docker_build/vripper-web.jar
docker build -t ghcr.io/dev-claw/vripper-web docker_build
docker tag ghcr.io/dev-claw/vripper-web:latest ghcr.io/dev-claw/vripper-web:${{ github.event.release.tag_name }}
docker push ghcr.io/dev-claw/vripper-web:latest
docker push ghcr.io/dev-claw/vripper-web:${{ github.event.release.tag_name }}
# End building docker image in ubuntu only
- name: Prepare Packaging
run: |
cp vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar jpackage/jar/vripper-gui.jar
@@ -57,9 +79,9 @@ jobs:
name: Package for Linux
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.png
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --type deb
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --type rpm
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.png
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type deb
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type rpm
mv dist/vripper-${{ github.event.release.tag_name }}-1.x86_64.rpm dist/vripper-linux-${{ github.event.release.tag_name }}.x86_64.rpm
mv dist/vripper_${{ github.event.release.tag_name }}-1_amd64.deb dist/vripper-linux-${{ github.event.release.tag_name }}_amd64.deb
@@ -67,9 +89,9 @@ jobs:
name: Package for Windows
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.ico
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --type msi
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --type exe
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.ico
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type msi
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type exe
cd dist
ren VRipper-${{ github.event.release.tag_name }}.msi vripper-windows-installer-${{ github.event.release.tag_name }}.msi
ren VRipper-${{ github.event.release.tag_name }}.exe vripper-windows-installer-${{ github.event.release.tag_name }}.exe
@@ -78,19 +100,19 @@ jobs:
name: Package for macOS(arm64)
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
- if: matrix.os == 'macos-13'
- if: matrix.os == 'macos-15-intel'
name: Package for macOS(x86_64)
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
@@ -120,8 +142,8 @@ jobs:
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip'
- if: matrix.os == 'macos-13'
- if: matrix.os == 'macos-15-intel'
name: Zip macOS(x86_64) portable
uses: thedoctor0/zip-release@0.7.1
with:
@@ -157,7 +179,7 @@ jobs:
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip
- if: matrix.os == 'macos-13'
- if: matrix.os == 'macos-15-intel'
name: Release packages for macOS(x86_64)
uses: softprops/action-gh-release@v1
with:
+4
View File
@@ -0,0 +1,4 @@
FROM azul/zulu-openjdk:25-latest
WORKDIR /opt/vripper-web
ENTRYPOINT ["java", "-jar", "vripper-web.jar"]
ADD vripper-web.jar /opt/vripper-web/vripper-web.jar
+104 -22
View File
@@ -15,14 +15,31 @@ ETH: 0xDdac82B16dC5E3D742fc915ffF583D8548A301cA
BTC: bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr
## Requirements
Direct access to `vipergirls.to` domain.
## Supported Image Hosts
The following hosts are supported:
* acidimg.cc
* imagetwist.com
* imagezilla.com
* imgspice.com
* imagebam.com
* imgbox.com
* imx.to
* pimpandhost.com
* pixhost.to
* pixxxels.cc
* turboimagehost.com
* postimg.cc
* imagevenue.com
* pixroute.to
* vipr.im
## Installing VRipper
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
:---:|:---:|:---:|:---:|:---:
[Installer (EXE)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-windows-installer-6.5.4.exe) <br /> [Installer (MSI)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-windows-installer-6.5.4.msi) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-windows-portable-6.5.4.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-macos-6.5.4.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-macos-6.5.4.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-macos-portable-6.5.4.x86_64.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-macos-6.5.4.arm64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-macos-6.5.4.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-macos-portable-6.5.4.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-linux-6.5.3_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-linux-6.5.4.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-linux-portable-6.5.4.zip) | [Java GUI (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-noarch-gui-6.5.4.jar) <br /> [Java Web (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.4/vripper-noarch-web-6.5.4.jar)
[Installer (EXE)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-windows-installer-6.10.0.exe) <br /> [Installer (MSI)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-windows-installer-6.10.0.msi) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-windows-portable-6.10.0.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-macos-6.10.0.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-macos-6.10.0.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-macos-portable-6.10.0.x86_64.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-macos-6.10.0.arm64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-macos-6.10.0.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-macos-portable-6.10.0.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-linux-6.5.3_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-linux-6.10.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-linux-portable-6.10.0.zip) | [Java GUI (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-noarch-gui-6.10.0.jar) <br /> [Java Web (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.10.0/vripper-noarch-web-6.10.0.jar)
Source code and previous versions are available on
the [Releases page](https://github.com/dev-claw/vripper-project/releases).
@@ -31,6 +48,90 @@ Application data (application logs, settings and persisted data) is stored in:
* Windows --> `C:\USERS\<your Windows username>\vripper`
* Linux and macOS --> `HOME_FOLDER/.config/vripper`
## Docker Support
The project is supplied as a container image and can be run locally or in production using Docker. By default:
- **gRPC server/client** uses port **30000**
- **Web UI** runs on port **8080**
### Run with `docker run`
```bash
docker run -d \
-p 30000:30000 \
-p 8080:8080 \
-e GRPC_ENABLED=true \
-e GRPC_PASSPHRASE=my-secure-secret \
ghcr.io/dev-claw/vripper-web:latest
```
### Run with Docker Compose
```yaml
services:
app:
image: ghcr.io/dev-claw/vripper-web:latest
environment:
- GRPC_ENABLED=true
- GRPC_PASSPHRASE=super-secret-passphrase
ports:
- "30000:30000"
- "8080:8080"
```
> **Tip:** For production, prefer storing the passphrase as a secret (see `Security Notes` below) rather than writing it
> directly in the Compose file.
### Environment Variables
The container supports these environment variables for enabling and securing the gRPC server.
### `GRPC_ENABLED`
- **Type:** String (interpreted as Boolean)
- **Accepted values:** `true` (case-insensitive) to enable; any other value or unset → disabled
- **Default:** `false` (gRPC disabled)
- **Behavior:**
- When `GRPC_ENABLED=true`, the application will attempt to start the gRPC server (default port: `30000`).
- When absent or set to anything other than `true`, the gRPC server will not be started.
- **Example (shell):**
```bash
export GRPC_ENABLED=true
```
- **Example (Docker Compose):**
```yaml
environment:
- GRPC_ENABLED=true
```
- **Notes / Best practices:**
- Treat the value as a simple on/off toggle. To avoid ambiguity, set explicitly to `true` or `false`.
- If you enable gRPC, make sure any required authentication (see `GRPC_PASSPHRASE`) is also configured.
### `GRPC_PASSPHRASE`
- **Type:** String
- **Default:** *no default* (empty / unset)
- **Required if:** `GRPC_ENABLED=true`
- **Purpose:** Shared secret used to authenticate gRPC clients. The application should validate that the passphrase is
present when gRPC is enabled and use it to verify client connections.
- **Example (shell):**
```bash
export GRPC_PASSPHRASE="my-very-strong-passphrase"
```
## Default Ports
- **gRPC server/client:** `30000`
- **Web UI:** `8080`
You can map these ports when running the container. The application supports overriding these defaults with environment
variables (`GRPC_PORT`, `SERVER_PORT`), configure them as needed.
## Important Note About Proxies
The use of proxies within VRipper is worthless, please stop using them **for now**. You will get **403 error** codes.
@@ -39,24 +140,6 @@ The use of proxies within VRipper is worthless, please stop using them **for now
If your ISP is blocking access to `vipergirls.to` domain, consider using a VPN or [Cloudflare WARP](https://one.one.one.one/) to bypass the block.
## Supported Image Hosts
The following hosts are supported:
* acidimg.cc
* imagetwist.com
* imagezilla.com
* imgspice.com
* imagebam.com
* imgbox.com
* imx.to
* pimpandhost.com
* pixhost.to
* pixxxels.cc
* turboimagehost.com
* postimg.cc
* imagevenue.com
* pixroute.to
* vipr.im
## Instructions to run from Jar file
You need Java 21+, you can download from https://adoptium.net/
@@ -72,7 +155,6 @@ For the WEB app
Application data (application logs, settings and persisted data) is stored in the location where you launched the jar for both GUI and WEB
## How to build
You need JDK 21 and a recent version of maven 3.8.x+
-1
View File
@@ -1,4 +1,3 @@
--icon icon.png
--java-options "-Dvripper.portable=false"
--linux-package-name vripper
--linux-app-release 1
-1
View File
@@ -1,4 +1,3 @@
--icon icon.icns
--java-options "-Dvripper.portable=false"
--mac-package-identifier me.mnlr.vripper.vripper-gui
--mac-package-name VRipper
-1
View File
@@ -1,4 +1,3 @@
--icon icon.ico
--java-options "-Dvripper.portable=false"
--win-dir-chooser
--win-menu
+10
View File
@@ -0,0 +1,10 @@
[Desktop Entry]
Name=VRipper
Comment=Image ripper tool for vipergirls
Exec=/opt/vripper/bin/VRipper
Icon=/opt/vripper/lib/VRipper.png
Terminal=false
Type=Application
Categories=Utility
MimeType=
StartupWMClass=me.vripper.gui.VripperGuiApplication

Before

Width:  |  Height:  |  Size: 303 KiB

After

Width:  |  Height:  |  Size: 303 KiB

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

+6 -5
View File
@@ -8,18 +8,19 @@
<packaging>pom</packaging>
<version>${revision}</version>
<properties>
<java.version>21</java.version>
<java.version>25</java.version>
<maven.deploy.skip>false</maven.deploy.skip>
<timestamp>${maven.build.timestamp}</timestamp>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss</maven.build.timestamp.format>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<revision>6.5.4</revision>
<kotlin.version>2.0.21</kotlin.version>
<revision>6.10.0</revision>
<app-version>6.10.0</app-version>
<kotlin.version>2.2.20</kotlin.version>
<slf4j.version>2.0.16</slf4j.version>
<logback.version>1.5.12</logback.version>
<kotlinx-coroutines.version>1.9.0</kotlinx-coroutines.version>
<kotlinx-coroutines.version>1.10.2</kotlinx-coroutines.version>
<exposed.version>0.55.0</exposed.version>
<snakeyaml.version>2.0</snakeyaml.version>
<koin.version>4.0.0</koin.version>
@@ -28,7 +29,7 @@
<failsafe.version>3.3.2</failsafe.version>
<caffeine.version>3.1.8</caffeine.version>
<jna-platform.version>5.14.0</jna-platform.version>
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
<kotlinx-serialization-json.version>1.9.0</kotlinx-serialization-json.version>
<httpclient5.version>5.4</httpclient5.version>
<httpcore5.version>5.3</httpcore5.version>
<liquibase-core.version>4.29.2</liquibase-core.version>
@@ -11,6 +11,8 @@ import me.vripper.data.repositories.impl.ThreadRepositoryImpl
import me.vripper.event.EventBus
import me.vripper.host.*
import me.vripper.services.*
import me.vripper.services.download.DownloadService
import me.vripper.services.download.QueueManager
import org.koin.core.qualifier.named
import org.koin.dsl.bind
import org.koin.dsl.module
@@ -37,8 +39,8 @@ val coreModule = module {
single<ThreadRepository> {
ThreadRepositoryImpl()
}
single<DataTransaction> {
DataTransaction(get(), get(), get(), get(), get(), get())
single<DataAccessService> {
DataAccessService(get(), get(), get(), get(), get(), get())
}
single<RetryPolicyService> {
RetryPolicyService()
@@ -52,6 +54,9 @@ val coreModule = module {
single<ThreadCacheService> {
ThreadCacheService(get())
}
single<QueueManager> {
QueueManager(get(), getAll(), get(), get(), get())
}
single<DownloadService> {
DownloadService(get(), get(), get(), get())
}
@@ -60,18 +65,18 @@ val coreModule = module {
}
single<AppEndpointService> {
AppEndpointService(get(), get(), get(), get(), get(), get())
AppEndpointService(get(), get(), get(), get(), get(), get(), get())
}
single((named("localAppEndpointService"))) {
AppEndpointService(get(), get(), get(), get(), get(), get())
AppEndpointService(get(), get(), get(), get(), get(), get(), get())
} bind IAppEndpointService::class
single<MetadataService> {
MetadataService(get(), get())
}
single {
AcidimgHost(get())
AcidimgHost()
} bind Host::class
single {
DPicMeHost()
@@ -6,15 +6,15 @@ import java.util.*
internal interface ImageRepository {
fun save(imageEntity: ImageEntity): ImageEntity
fun save(imageEntityList: List<ImageEntity>)
fun deleteAllByPostId(postId: Long)
fun findByPostId(postId: Long): List<ImageEntity>
fun deleteAllByPostEntityId(postEntityId: Long)
fun findByPostEntityId(postEntityId: Long): List<ImageEntity>
fun countError(): Int
fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity>
fun stopByPostIdAndIsNotCompleted(postId: Long): Int
fun stopByPostIdAndIsNotCompleted(): Int
fun findByPostIdAndIsError(postId: Long): List<ImageEntity>
fun findByPostEntityIdAndIsNotCompleted(postEntityId: Long): List<ImageEntity>
fun stopByPostEntityIdAndIsNotCompleted(postEntityId: Long): Int
fun stopByPostEntityIdAndIsNotCompleted(): Int
fun findByPostEntityIdAndIsError(postEntityId: Long): List<ImageEntity>
fun findById(id: Long): Optional<ImageEntity>
fun update(imageEntity: ImageEntity)
fun update(imageEntities: List<ImageEntity>)
fun deleteAllByPostId(postIds: List<Long>)
fun deleteAllByPostEntityId(postEntityIds: List<Long>)
}
@@ -5,7 +5,7 @@ import java.util.*
internal interface MetadataRepository {
fun save(metadataEntity: MetadataEntity): MetadataEntity
fun findByPostId(postId: Long): Optional<MetadataEntity>
fun deleteByPostId(postId: Long): Int
fun deleteAllByPostId(postIds: List<Long>)
fun findByPostEntityId(postEntityId: Long): Optional<MetadataEntity>
fun deleteByPostEntityId(postEntityId: Long): Int
fun deleteAllByPostEntityId(postEntityIds: List<Long>)
}
@@ -4,17 +4,16 @@ import me.vripper.entities.PostEntity
internal interface PostRepository {
fun save(postEntities: List<PostEntity>): List<PostEntity>
fun findByPostId(postId: Long): PostEntity?
fun findById(id: Long): PostEntity?
fun findCompleted(): List<Long>
fun findAll(): List<PostEntity>
fun existByPostEntityId(postEntityId: Long): Boolean
fun existByPostId(postId: Long): Boolean
fun setDownloadingToStopped(): Int
fun deleteByPostId(postId: Long): Int
fun deleteByPostEntityId(postEntityId: Long): Int
fun update(postEntity: PostEntity)
fun update(postEntities: List<PostEntity>)
fun findMaxRank(): Int?
fun deleteAll(postIds: List<Long>)
fun deleteAll(postEntityIds: List<Long>)
fun stopAll()
fun findAllNonCompletedPostIds(): List<Long>
fun findAllNonCompletedPostEntityIds(): List<Long>
}
@@ -16,12 +16,11 @@ internal class ImageRepositoryImpl : ImageRepository {
it[downloaded] = imageEntity.downloaded
it[host] = imageEntity.host
it[index] = imageEntity.index
it[postId] = imageEntity.postId
it[status] = imageEntity.status.name
it[size] = imageEntity.size
it[url] = imageEntity.url
it[thumbUrl] = imageEntity.thumbUrl
it[postIdRef] = imageEntity.postIdRef
it[postIdRef] = imageEntity.postEntityId
it[filename] = imageEntity.filename
}.value
return imageEntity.copy(id = id)
@@ -32,23 +31,22 @@ internal class ImageRepositoryImpl : ImageRepository {
this[ImageTable.downloaded] = it.downloaded
this[ImageTable.host] = it.host
this[ImageTable.index] = it.index
this[ImageTable.postId] = it.postId
this[ImageTable.status] = it.status.name
this[ImageTable.size] = it.size
this[ImageTable.url] = it.url
this[ImageTable.thumbUrl] = it.thumbUrl
this[ImageTable.postIdRef] = it.postIdRef
this[ImageTable.postIdRef] = it.postEntityId
this[ImageTable.filename] = it.filename
}
}
override fun deleteAllByPostId(postId: Long) {
ImageTable.deleteWhere { ImageTable.postId eq postId }
override fun deleteAllByPostEntityId(postEntityId: Long) {
ImageTable.deleteWhere { ImageTable.postIdRef eq postEntityId }
}
override fun findByPostId(postId: Long): List<ImageEntity> {
override fun findByPostEntityId(postEntityId: Long): List<ImageEntity> {
return ImageTable.selectAll().where {
ImageTable.postId eq postId
ImageTable.postIdRef eq postEntityId
}.map(this::transform)
}
@@ -59,28 +57,28 @@ internal class ImageRepositoryImpl : ImageRepository {
.count().toInt()
}
override fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity> {
override fun findByPostEntityIdAndIsNotCompleted(postEntityId: Long): List<ImageEntity> {
return ImageTable
.selectAll().where {
(ImageTable.postId eq postId) and (ImageTable.status neq Status.FINISHED.name)
(ImageTable.postIdRef eq postEntityId) and (ImageTable.status neq Status.FINISHED.name)
}.map(this::transform)
}
override fun stopByPostIdAndIsNotCompleted(postId: Long): Int {
return ImageTable.update({ (ImageTable.postId eq postId) and (ImageTable.status neq Status.FINISHED.name) }) {
override fun stopByPostEntityIdAndIsNotCompleted(postEntityId: Long): Int {
return ImageTable.update({ (ImageTable.postIdRef eq postEntityId) and (ImageTable.status neq Status.FINISHED.name) }) {
it[status] = Status.STOPPED.name
}
}
override fun stopByPostIdAndIsNotCompleted(): Int {
override fun stopByPostEntityIdAndIsNotCompleted(): Int {
return ImageTable.update({ (ImageTable.status neq Status.FINISHED.name) }) {
it[status] = Status.STOPPED.name
}
}
override fun findByPostIdAndIsError(postId: Long): List<ImageEntity> {
override fun findByPostEntityIdAndIsError(postEntityId: Long): List<ImageEntity> {
return ImageTable.selectAll().where {
(ImageTable.postId eq postId) and (ImageTable.status eq Status.ERROR.name)
(ImageTable.postIdRef eq postEntityId) and (ImageTable.status eq Status.ERROR.name)
}.map(this::transform)
}
@@ -110,32 +108,31 @@ internal class ImageRepositoryImpl : ImageRepository {
this[ImageTable.downloaded] = it.downloaded
this[ImageTable.host] = it.host
this[ImageTable.index] = it.index
this[ImageTable.postId] = it.postId
this[ImageTable.status] = it.status.name
this[ImageTable.size] = it.size
this[ImageTable.url] = it.url
this[ImageTable.thumbUrl] = it.thumbUrl
this[ImageTable.postIdRef] = it.postIdRef
this[ImageTable.postIdRef] = it.postEntityId
this[ImageTable.filename] = it.filename
}
}
override fun deleteAllByPostId(postIds: List<Long>) {
override fun deleteAllByPostEntityId(postEntityIds: List<Long>) {
val conn = TransactionManager.current().connection.connection as Connection
conn.prepareStatement("CREATE TEMPORARY TABLE IMAGES_DELETE(POST_ID BIGINT PRIMARY KEY)")
conn.prepareStatement("CREATE TEMPORARY TABLE IMAGES_DELETE(POST_ID_REF BIGINT PRIMARY KEY)")
.use {
it.execute()
}
conn.prepareStatement("INSERT INTO IMAGES_DELETE VALUES ( ? )").use { ps ->
postIds.forEach {
postEntityIds.forEach {
ps.setLong(1, it)
ps.addBatch()
}
ps.executeBatch()
}
conn.prepareStatement("DELETE FROM IMAGE WHERE POST_ID IN (SELECT POST_ID FROM IMAGES_DELETE)")
conn.prepareStatement("DELETE FROM IMAGE WHERE POST_ID_REF IN (SELECT POST_ID_REF FROM IMAGES_DELETE)")
.use {
it.execute()
}
@@ -147,7 +144,6 @@ internal class ImageRepositoryImpl : ImageRepository {
private fun transform(resultRow: ResultRow): ImageEntity {
val id = resultRow[ImageTable.id].value
val postId = resultRow[ImageTable.postId]
val url = resultRow[ImageTable.url]
val thumbUrl = resultRow[ImageTable.thumbUrl]
val host = resultRow[ImageTable.host]
@@ -159,7 +155,6 @@ internal class ImageRepositoryImpl : ImageRepository {
val filename = resultRow[ImageTable.filename]
return ImageEntity(
id,
postId,
url,
thumbUrl,
host,
@@ -18,17 +18,17 @@ internal class MetadataRepositoryImpl : MetadataRepository {
override fun save(metadataEntity: MetadataEntity): MetadataEntity {
MetadataTable.insert {
it[postId] = metadataEntity.postId
it[postIdRef] = metadataEntity.postIdRef
it[data] = Json.encodeToString(metadataEntity.data)
}
return metadataEntity
}
override fun findByPostId(postId: Long): Optional<MetadataEntity> {
override fun findByPostEntityId(postEntityId: Long): Optional<MetadataEntity> {
val result = MetadataTable.selectAll().where {
MetadataTable.postId eq postId
MetadataTable.postIdRef eq postEntityId
}.map(::transform)
return if (result.isEmpty()) {
@@ -39,31 +39,31 @@ internal class MetadataRepositoryImpl : MetadataRepository {
}
private fun transform(row: ResultRow): MetadataEntity {
val id = row[MetadataTable.postId]
val postEntityId = row[MetadataTable.postIdRef]
val data = Json.decodeFromString(row[MetadataTable.data]) as MetadataEntity.Data
return MetadataEntity(id, data)
return MetadataEntity(postEntityId, data)
}
override fun deleteByPostId(postId: Long): Int {
return MetadataTable.deleteWhere { MetadataTable.postId eq postId }
override fun deleteByPostEntityId(postEntityId: Long): Int {
return MetadataTable.deleteWhere { MetadataTable.postIdRef eq postEntityId }
}
override fun deleteAllByPostId(postIds: List<Long>) {
override fun deleteAllByPostEntityId(postEntityIds: List<Long>) {
val conn = TransactionManager.current().connection.connection as Connection
conn.prepareStatement("CREATE TEMPORARY TABLE METADATA_DELETE(POST_ID BIGINT PRIMARY KEY)")
conn.prepareStatement("CREATE TEMPORARY TABLE METADATA_DELETE(POST_ID_REF BIGINT PRIMARY KEY)")
.use {
it.execute()
}
conn.prepareStatement("INSERT INTO METADATA_DELETE VALUES ( ? )").use { ps ->
postIds.forEach {
postEntityIds.forEach {
ps.setLong(1, it)
ps.addBatch()
}
ps.executeBatch()
}
conn.prepareStatement("DELETE FROM METADATA WHERE POST_ID IN (SELECT POST_ID FROM METADATA_DELETE)")
conn.prepareStatement("DELETE FROM METADATA WHERE POST_ID_REF IN (SELECT POST_ID_REF FROM METADATA_DELETE)")
.use {
it.execute()
}
@@ -19,12 +19,11 @@ internal class PostRepositoryImpl :
this[PostTable.status] = post.status.name
this[PostTable.done] = post.done
this[PostTable.total] = post.total
this[PostTable.rank] = post.rank
this[PostTable.hosts] = post.hosts.joinToString(delimiter)
this[PostTable.outputPath] = post.downloadDirectory
this[PostTable.folderName] = post.folderName
this[PostTable.postId] = post.postId
this[PostTable.threadId] = post.threadId
this[PostTable.postId] = post.vgPostId
this[PostTable.threadId] = post.vgThreadId
this[PostTable.postTitle] = post.postTitle
this[PostTable.threadTitle] = post.threadTitle
this[PostTable.forum] = post.forum
@@ -36,17 +35,10 @@ internal class PostRepositoryImpl :
}.map(::transform)
}
override fun findByPostId(postId: Long): PostEntity? {
val result = PostTable.selectAll().where {
PostTable.postId eq postId
}.map(::transform)
return result.firstOrNull()
}
override fun findCompleted(): List<Long> {
return PostTable.select(PostTable.postId).where {
return PostTable.select(PostTable.id).where {
(PostTable.status eq Status.FINISHED.name) and (PostTable.done greaterEq PostTable.total)
}.map { it[PostTable.postId] }
}.map { it[PostTable.id].value }
}
override fun findById(id: Long): PostEntity? {
@@ -60,8 +52,12 @@ internal class PostRepositoryImpl :
return PostTable.selectAll().map { transform(it) }
}
override fun existByPostEntityId(postEntityId: Long): Boolean {
return PostTable.select(PostTable.id).where { PostTable.id eq postEntityId }.count() > 0
}
override fun existByPostId(postId: Long): Boolean {
return PostTable.select(PostTable.id).where { PostTable.postId eq postId }.count() > 0
return PostTable.select(PostTable.postId).where { PostTable.postId eq postId }.count() > 0
}
override fun setDownloadingToStopped(): Int {
@@ -70,15 +66,14 @@ internal class PostRepositoryImpl :
}
}
override fun deleteByPostId(postId: Long): Int {
return PostTable.deleteWhere { PostTable.postId eq postId }
override fun deleteByPostEntityId(postEntityId: Long): Int {
return PostTable.deleteWhere { PostTable.id eq postEntityId }
}
override fun update(postEntity: PostEntity) {
PostTable.update({ PostTable.id eq postEntity.id }) {
it[status] = postEntity.status.name
it[done] = postEntity.done
it[rank] = postEntity.rank
it[size] = postEntity.size
it[downloaded] = postEntity.downloaded
it[folderName] = postEntity.folderName
@@ -91,12 +86,11 @@ internal class PostRepositoryImpl :
this[PostTable.status] = post.status.name
this[PostTable.done] = post.done
this[PostTable.total] = post.total
this[PostTable.rank] = post.rank
this[PostTable.hosts] = post.hosts.joinToString(delimiter)
this[PostTable.outputPath] = post.downloadDirectory
this[PostTable.folderName] = post.folderName
this[PostTable.postId] = post.postId
this[PostTable.threadId] = post.threadId
this[PostTable.postId] = post.vgPostId
this[PostTable.threadId] = post.vgThreadId
this[PostTable.postTitle] = post.postTitle
this[PostTable.threadTitle] = post.threadTitle
this[PostTable.forum] = post.forum
@@ -104,31 +98,26 @@ internal class PostRepositoryImpl :
this[PostTable.token] = post.token
this[PostTable.size] = post.size
this[PostTable.downloaded] = post.downloaded
this[PostTable.addedAt] = post.addedOn
}
}
override fun findMaxRank(): Int? {
return PostTable.select(PostTable.rank.max())
.firstOrNull()
?.get(PostTable.rank.max())
}
override fun deleteAll(postIds: List<Long>) {
override fun deleteAll(postEntityIds: List<Long>) {
val conn = TransactionManager.current().connection.connection as Connection
conn.prepareStatement("CREATE TEMPORARY TABLE POSTS_DELETE(POST_ID BIGINT PRIMARY KEY)")
conn.prepareStatement("CREATE TEMPORARY TABLE POSTS_DELETE(ID BIGINT PRIMARY KEY)")
.use {
it.execute()
}
conn.prepareStatement("INSERT INTO POSTS_DELETE VALUES ( ? )").use { ps ->
postIds.forEach {
postEntityIds.forEach {
ps.setLong(1, it)
ps.addBatch()
}
ps.executeBatch()
}
conn.prepareStatement("DELETE FROM POST WHERE POST_ID IN (SELECT POST_ID FROM POSTS_DELETE)")
conn.prepareStatement("DELETE FROM POST WHERE ID IN (SELECT ID FROM POSTS_DELETE)")
.use {
it.execute()
}
@@ -144,10 +133,10 @@ internal class PostRepositoryImpl :
}
}
override fun findAllNonCompletedPostIds(): List<Long> {
return PostTable.select(PostTable.postId).where {
override fun findAllNonCompletedPostEntityIds(): List<Long> {
return PostTable.select(PostTable.id).where {
PostTable.status neq Status.FINISHED.name
}.map { it[PostTable.postId] }
}.map { it[PostTable.id].value }
}
private fun transform(resultRow: ResultRow): PostEntity {
@@ -167,7 +156,6 @@ internal class PostRepositoryImpl :
val downloadDirectory = resultRow[PostTable.outputPath]
val folderName = resultRow[PostTable.folderName]
val addedOn = resultRow[PostTable.addedAt]
val rank = resultRow[PostTable.rank]
val size = resultRow[PostTable.size]
val downloaded = resultRow[PostTable.downloaded]
return PostEntity(
@@ -186,7 +174,6 @@ internal class PostRepositoryImpl :
folderName,
status,
done,
rank,
size,
downloaded
)
@@ -6,7 +6,6 @@ internal object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") {
val downloaded = long("DOWNLOADED")
val host = byte("HOST")
val index = integer("INDEX")
val postId = long("POST_ID")
val status = varchar("STATUS", 15)
val filename = varchar("FILENAME", 260)
val size = long("SIZE")
@@ -3,6 +3,6 @@ package me.vripper.data.tables
import org.jetbrains.exposed.sql.Table
internal object MetadataTable : Table(name = "METADATA") {
val postId = long("POST_ID")
val postIdRef = long("POST_ID_REF").references(PostTable.id, fkName = "METADATA_POST_ID_REF_POST_ID_FK")
val data = varchar("DATA", 1_000_000)
}
@@ -5,29 +5,13 @@ import kotlinx.serialization.Serializable
@Serializable
data class ImageEntity(
val id: Long = -1,
val postId: Long,
val url: String,
val thumbUrl: String,
val host: Byte,
val index: Int,
val postIdRef: Long = -1,
val postEntityId: Long = -1,
var size: Long = -1,
var downloaded: Long = 0,
var status: Status = Status.STOPPED,
var filename: String = "",
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as ImageEntity
if (url != other.url) return false
return true
}
override fun hashCode(): Int {
return url.hashCode()
}
}
)
@@ -3,7 +3,7 @@ package me.vripper.entities
import kotlinx.serialization.Serializable
@Serializable
data class MetadataEntity(val postId: Long, val data: Data) {
data class MetadataEntity(val postIdRef: Long, val data: Data) {
@Serializable
data class Data(
val postedBy: String,
@@ -12,8 +12,8 @@ data class PostEntity(
val forum: String,
val url: String,
val token: String,
val postId: Long,
val threadId: Long,
val vgPostId: Long,
val vgThreadId: Long,
val total: Int,
val hosts: Set<String>,
val downloadDirectory: String,
@@ -21,21 +21,6 @@ data class PostEntity(
var folderName: String,
var status: Status = Status.STOPPED,
var done: Int = 0,
var rank: Int = Int.MAX_VALUE,
var size: Long = -1,
var downloaded: Long = 0,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as PostEntity
return postId == other.postId
}
override fun hashCode(): Int {
return postId.hashCode()
}
}
)
@@ -9,17 +9,4 @@ data class ThreadEntity(
val link: String,
val threadId: Long,
var total: Int = 0,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as ThreadEntity
return threadId == other.threadId
}
override fun hashCode(): Int {
return threadId.hashCode()
}
}
)
@@ -8,7 +8,7 @@ import me.vripper.model.Settings
data class PostCreateEvent(val postEntities: List<PostEntity>)
data class PostUpdateEvent(val postEntities: List<PostEntity>)
data class PostDeleteEvent(val postIds: List<Long>)
data class PostDeleteEvent(val postEntityIds: List<Long>)
data class ImageEvent(val imageEntities: List<ImageEntity>)
data class ThreadCreateEvent(val threadEntity: ThreadEntity)
data class ThreadUpdateEvent(val threadEntity: ThreadEntity)
@@ -1,10 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.HTTPService
import me.vripper.services.download.ImageDownloadRunnable.Context
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -13,29 +11,27 @@ import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.core5.http.message.BasicNameValuePair
import org.w3c.dom.Node
internal class AcidimgHost(
private val httpService: HTTPService,
) : Host("acidimg.cc", 0) {
internal class AcidimgHost : Host("acidimg.cc", 0) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
image: ImageEntity, context: ImageDownloadContext
context: Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
try {
log.debug(
String.format(
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, image.url
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, context.imageEntity.url
)
)
XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH)
} catch (e: XpathException) {
throw HostException(e)
}
log.debug(String.format("Click button found for %s", image.url))
val httpPost = HttpPost(image.url).also {
it.addHeader("Referer", image.url)
log.debug(String.format("Click button found for %s", context.imageEntity.url))
val httpPost = HttpPost(context.imageEntity.url).also {
it.addHeader("Referer", context.imageEntity.url)
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair(
@@ -44,8 +40,9 @@ internal class AcidimgHost(
)
)
)
it.setAbsoluteRequestUri(true)
}.also { context.requests.add(it) }
log.debug(String.format("Requesting %s", httpPost.uri))
log.info("{}", httpPost)
val doc = try {
httpService.client.execute(
httpPost, context.httpContext
@@ -58,17 +55,17 @@ internal class AcidimgHost(
throw HostException(e)
}
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.url
"Xpath '%s' cannot be found in '%s'", IMG_XPATH, context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim()
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim()
Pair(
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -14,12 +13,11 @@ internal class DPicMeHost : Host("dpic.me", 1) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -27,11 +25,11 @@ internal class DPicMeHost : Host("dpic.me", 1) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle =
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { e: Node -> e.textContent.trim() }
@@ -4,13 +4,12 @@ import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.runBlocking
import me.vripper.entities.ImageEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DataAccessService
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.services.download.ImageDownloadRunnable.Context
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
@@ -32,8 +31,8 @@ internal abstract class Host(
) : KoinComponent {
private val log by LoggerDelegate()
private val httpService: HTTPService by inject()
private val dataTransaction: DataTransaction by inject()
protected val httpService: HTTPService by inject()
private val dataAccessService: DataAccessService by inject()
private val downloadSpeedService: DownloadSpeedService by inject()
companion object {
@@ -49,44 +48,41 @@ internal abstract class Host(
hosts[hostName] = hostId
}
@Throws(HostException::class)
abstract fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: Context
): Pair<String, String>
@Throws(HostException::class)
fun downloadInternal(image: ImageEntity, context: ImageDownloadContext): DownloadedImage {
fun downloadInternal(context: Context): DownloadedImage {
if (hostId == 8.toByte()) {
return downloadByHost(image, context)
return downloadByHost(context)
}
val headers = head(image.url, context)
val headers = head(context)
// is the body of type image ?
val imageMimeType = getImageMimeType(headers)
val downloadedImage = if (imageMimeType != null) {
// a direct link, awesome
val downloadedImage = fetch(image.url, context) {
val downloadedImage = fetch(context.imageEntity.url, context) {
handleImageDownload(it, context)
}
DownloadedImage(getDefaultImageName(image.url), downloadedImage.first, downloadedImage.second)
DownloadedImage(getDefaultImageName(context.imageEntity.url), downloadedImage.first, downloadedImage.second)
} else {
// linked image ?
val value = headers.find { it.name.contains("content-type", true) }?.value
if (value != null) {
if (value.contains("text/html")) {
downloadByHost(image, context)
downloadByHost(context)
} else {
throw HostException("Unable to download ${image.url}, can't process content type $value")
throw HostException("Unable to download ${context.imageEntity.url}, can't process content type $value")
}
} else {
throw HostException("Unexpected server response for ${image.url}, response have no content type")
throw HostException("Unexpected server response for ${context.imageEntity.url}, response have no content type")
}
}
return downloadedImage
}
private fun downloadByHost(image: ImageEntity, context: ImageDownloadContext): DownloadedImage {
val resolvedImage = resolve(image, context)
private fun downloadByHost(context: Context): DownloadedImage {
val resolvedImage = resolve(context)
val downloadImage: Pair<Path, ImageMimeType> =
fetch(resolvedImage.second, context) {
handleImageDownload(it, context)
@@ -96,7 +92,7 @@ internal abstract class Host(
private fun handleImageDownload(
response: ClassicHttpResponse,
context: ImageDownloadContext
context: Context
): Pair<Path, ImageMimeType> {
val mimeType = getImageMimeType(response.headers)
?: throw HostException("Unsupported image type ${response.getFirstHeader("content-type")}")
@@ -107,23 +103,22 @@ internal abstract class Host(
".tmp"
)
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
val image = context.imageEntity
synchronized(image.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val size = if (image.size < 0) {
synchronized(context.imageEntity.postEntityId.toString().intern()) {
val post = dataAccessService.findPostByEntityId(context.imageEntity.postEntityId)
val size = if (context.imageEntity.size < 0) {
response.entity.contentLength
} else {
0
}
image.size = response.entity.contentLength
context.imageEntity.size = response.entity.contentLength
post.size += size
transaction {
dataTransaction.updateImage(image)
dataTransaction.updatePost(post)
dataAccessService.updateImage(context.imageEntity)
dataAccessService.updatePost(post)
}
}
log.debug(
"Length is ${image.size}"
"Length is ${context.imageEntity.size}"
)
log.debug(
"Starting data transfer"
@@ -132,7 +127,7 @@ internal abstract class Host(
var read: Int
val reporterJob = context.launchCoroutine {
while (isActive) {
dataTransaction.updateImage(image, false)
dataAccessService.updateImage(context.imageEntity, false)
delay(100)
}
}
@@ -140,13 +135,13 @@ internal abstract class Host(
.also { read = it } != -1
) {
bos.write(buffer, 0, read)
image.downloaded += read
context.imageEntity.downloaded += read
downloadSpeedService.reportDownloadedBytes(read.toLong())
}
runBlocking {
reporterJob.cancelAndJoin()
}
dataTransaction.updateImage(image)
dataAccessService.updateImage(context.imageEntity)
Pair(tempImage, mimeType)
}
}
@@ -155,10 +150,13 @@ internal abstract class Host(
return url.contains(hostName)
}
@Throws(HostException::class)
fun head(url: String, context: ImageDownloadContext): Array<Header> {
val httpHead = HttpHead(url).also { context.requests.add(it) }
log.debug(String.format("Requesting %s", url))
fun head(context: Context): Array<Header> {
val httpHead = HttpHead(context.imageEntity.url).also {
it.addHeader("Referer", "https://vipergirls.to/")
it.setAbsoluteRequestUri(true)
context.requests.add(it)
}
log.info("{}", httpHead)
return httpService.client.execute(
httpHead,
context.httpContext
@@ -170,15 +168,18 @@ internal abstract class Host(
}
}
@Throws(HostException::class)
fun <T> fetch(
url: String,
context: ImageDownloadContext,
context: Context,
transformer: (ClassicHttpResponse) -> T
): T {
val referer = context.headers["Referer"] ?: "https://vipergirls.to/"
val httpGet =
HttpGet(url).also { it.addHeader("Referer", context.imageEntity.url) }.also { context.requests.add(it) }
log.debug(String.format("Requesting %s", url))
HttpGet(url).also {
it.addHeader("Referer", referer)
it.setAbsoluteRequestUri(true)
}.also { context.requests.add(it) }
log.info("{}", httpGet)
return httpService.client.execute(httpGet, context.httpContext) {
if (it.code / 100 != 2) {
throw DownloadException("Server returned code ${it.code}")
@@ -189,13 +190,13 @@ internal abstract class Host(
fun fetchDocument(
url: String,
context: ImageDownloadContext
context: Context
): Document {
return fetch(url, context) {
HtmlUtils.clean(it.entity.content)
}.also {
if (log.isDebugEnabled) {
log.debug("Cleaning $url response", url)
log.debug("Cleaning {} response", url)
}
}
}
@@ -1,15 +1,13 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.w3c.dom.Node
import java.sql.Date
import java.time.LocalDateTime
import java.time.ZoneId
import java.util.*
@@ -19,22 +17,26 @@ internal class ImageBamHost : Host("imagebam.com", 2) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val doc = try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, context.imageEntity.url))
if (XpathUtils.getAsNode(document, CONTINUE_XPATH) != null) {
val clientCookie = BasicClientCookie("nsfw_inter", "1")
clientCookie.domain = "www.imagebam.com"
clientCookie.path = "/"
clientCookie.expiryDate =
Date.from(
LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant()
)
context.httpContext.cookieStore.addCookie(clientCookie)
fetch(image.url, context) {
val nsfwCookie = BasicClientCookie("nsfw_inter", "1").apply {
domain = "www.imagebam.com"
path = "/"
setExpiryDate(LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant())
}
val sfwCookie = BasicClientCookie("sfw_inter", "1").apply {
domain = "www.imagebam.com"
path = "/"
setExpiryDate(LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant())
}
context.httpContext.cookieStore.addCookie(nsfwCookie)
context.httpContext.cookieStore.addCookie(sfwCookie)
fetch(context.imageEntity.url, context) {
HtmlUtils.clean(it.entity.content)
}
} else {
@@ -44,7 +46,7 @@ internal class ImageBamHost : Host("imagebam.com", 2) {
throw HostException(e)
}
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -52,11 +54,11 @@ internal class ImageBamHost : Host("imagebam.com", 2) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { e: Node -> e.textContent.trim { it <= ' ' } }
.orElse("")
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -14,12 +13,11 @@ internal class ImageTwistHost : Host("imagetwist.com", 3) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -27,11 +25,11 @@ internal class ImageTwistHost : Host("imagetwist.com", 3) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle =
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { obj: Node -> obj.textContent }
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -14,21 +13,20 @@ internal class ImageVenueHost : Host("imagevenue.com", 4) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val doc = try {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
log.debug(
String.format(
"Looking for xpath expression %s in %s",
CONTINUE_BUTTON_XPATH,
image.url
context.imageEntity.url
)
)
if (XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
fetch(image.url, context) {
fetch(context.imageEntity.url, context) {
HtmlUtils.clean(it.entity.content)
}
} else {
@@ -38,7 +36,7 @@ internal class ImageVenueHost : Host("imagevenue.com", 4) {
throw HostException(e)
}
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -46,11 +44,11 @@ internal class ImageVenueHost : Host("imagevenue.com", 4) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -12,12 +11,11 @@ internal class ImageZillaHost : Host("imagezilla.net", 5) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val titleNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -25,17 +23,17 @@ internal class ImageZillaHost : Host("imagezilla.net", 5) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
log.debug(String.format("Resolving name for %s", image.url))
log.debug(String.format("Resolving name for %s", context.imageEntity.url))
var title = titleNode.attributes.getNamedItem("title").textContent.trim()
titleNode.textContent.trim()
if (title.isEmpty()) {
title = getDefaultImageName(image.url)
title = getDefaultImageName(context.imageEntity.url)
}
return try {
Pair(title, image.url.replace("show", "images"))
Pair(title, context.imageEntity.url.replace("show", "images"))
} catch (e: Exception) {
throw HostException("Unexpected error occurred", e)
}
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -13,12 +12,11 @@ internal class ImgSpiceHost : Host("imgspice.com", 7) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -26,11 +24,11 @@ internal class ImgSpiceHost : Host("imgspice.com", 7) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -13,12 +12,11 @@ internal class ImgboxHost : Host("imgbox.com", 6) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -26,11 +24,11 @@ internal class ImgboxHost : Host("imgbox.com", 6) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("title").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(imgTitle, imgUrl)
@@ -1,71 +1,95 @@
package me.vripper.host
import com.github.benmanes.caffeine.cache.Cache
import com.github.benmanes.caffeine.cache.Caffeine
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.utilities.ApplicationProperties.IMX_SUBDOMAINS
import me.vripper.model.HostName
import me.vripper.model.HostSettingKey
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import org.apache.hc.client5.http.classic.methods.HttpHead
import org.apache.hc.client5.http.config.ConnectionConfig
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
import java.util.concurrent.TimeUnit
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.core5.http.message.BasicNameValuePair
internal class ImxHost : Host("imx.to", 8) {
companion object {
private val resolvedHosts: Cache<Long, String> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build()
}
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
log.debug("Resolving name and image url for ${image.url}")
val imgTitle = String.format("IMG_%04d", image.index + 1)
synchronized(image.postId.toString().intern()) {
val resolvedHost = resolvedHosts.getIfPresent(image.postId)
if (resolvedHost != null) {
val imgUrl = image.thumbUrl.replace("imx.to", resolvedHost).replace("u/t/", "i/")
.replace("t/", "i/")
return Pair(
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
)
} else {
IMX_SUBDOMAINS.forEach { subDomain ->
val imgUrl = image.thumbUrl.replace("imx.to", subDomain).replace("u/t/", "i/").replace("t/", "i/")
val result = runCatching {
RequestLimit.getPermit(1)
val httpHead = HttpHead(imgUrl).also { context.requests.add(it) }
HttpClients.custom().apply {
setConnectionManager(BasicHttpClientConnectionManager().apply {
connectionConfig = ConnectionConfig.custom()
.setConnectTimeout(5000, TimeUnit.MILLISECONDS)
.setSocketTimeout(5000, TimeUnit.MILLISECONDS)
.build()
})
}.build().execute(httpHead) { response ->
if (response.code / 100 != 2) {
throw HostException("Invalid response")
}
}
}
if (result.isSuccess) {
resolvedHosts.put(image.postId, subDomain)
return Pair(
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
)
}
}
log.debug("Resolving name and image url for ${context.imageEntity.url}")
val imgTitle = getTitle(context)
val imgUrl = findPattern(context.imageEntity)
return Pair(
imgTitle, imgUrl
)
}
private fun getTitle(context: ImageDownloadRunnable.Context): String {
return if (context.settings.hostSettings[HostName.IMX]?.get(HostSettingKey.TRY_TO_FETCH_ORIGINAL_FILENAME)
.toBoolean()
) {
val httpsUrl = context.imageEntity.url.replace("http:", "https:")
val document = fetchDocument(httpsUrl, context)
var value: String? = null
log.debug("Looking for xpath expression $CONTINUE_BUTTON_XPATH in $httpsUrl")
val contDiv = XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH)
?: throw HostException("$CONTINUE_BUTTON_XPATH cannot be found")
val node = contDiv.attributes.getNamedItem("value")
if (node != null) {
value = node.textContent
}
log.debug("Click button found for $httpsUrl")
val httpPost: HttpPost = HttpPost(httpsUrl).also {
it.entity = UrlEncodedFormEntity(listOf(BasicNameValuePair("imgContinue", value)))
}.also { context.requests.add(it) }
log.debug("Requesting {}", httpPost)
val doc = httpService.client.execute(
httpPost, context.httpContext
) { response ->
log.debug("Cleaning response for {}", httpPost)
HtmlUtils.clean(response.entity.content)
}
log.debug("Looking for xpath expression $IMG_XPATH in $httpsUrl")
val imgNode = XpathUtils.getAsNode(doc, IMG_XPATH)
log.debug("Resolving name for $httpsUrl")
val imgTitle = imgNode?.attributes?.getNamedItem("alt")?.textContent?.trim() ?: ""
imgTitle
} else {
getDefaultImageName(context.imageEntity.thumbUrl)
}
throw HostException("Unable to find full size image for ${image.url}")
}
private fun findPattern(image: ImageEntity): String {
val url = image.thumbUrl
.replace("http:", "https:")
return if (url.startsWith("https://image.imx.to/u/t/")) {
"https://image.imx.to/u/i/" + url.replace("https://image.imx.to/u/t/", "")
} else if (url.startsWith("https://imx.to/u/t")) {
"https://image.imx.to/u/i/" + url.replace("https://imx.to/u/t", "")
} else if (url.startsWith("https://t.imx.to/t/")) {
"https://image.imx.to/u/i/" + url.replace("https://t.imx.to/t/", "")
} else if (url.startsWith("https://imx.to/upload/small/")) {
"https://image.imx.to/u/i/" + url.replace("https://imx.to/upload/small/", "")
} else if (url.startsWith("https://i.imx.to/t/")) {
"https://image.imx.to/u/i/" + url.replace("https://i.imx.to/t/", "")
} else if (url.startsWith("https://image.imx.to/u/i/")) {
url
} else {
throw HostException("Cannot find pattern for url ${image.thumbUrl}")
}
}
companion object {
private const val CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']"
private const val IMG_XPATH = "//img[@class='centred']"
}
}
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -16,13 +15,12 @@ internal class PimpandhostHost : Host("pimpandhost.com", 9) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val newUrl: String
try {
newUrl = appendUri(
image.url.replace("http://", "https://").replace("-medium(\\.html)?".toRegex(), ""),
context.imageEntity.url.replace("http://", "https://").replace("-medium(\\.html)?".toRegex(), ""),
"size=original"
)
} catch (e: Exception) {
@@ -40,7 +38,7 @@ internal class PimpandhostHost : Host("pimpandhost.com", 9) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -13,12 +12,11 @@ internal class PixRouteHost : Host("pixroute.com", 11) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -26,11 +24,11 @@ internal class PixRouteHost : Host("pixroute.com", 11) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
Pair(imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' },
imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
} catch (e: Exception) {
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -13,12 +12,11 @@ internal class PixhostHost : Host("pixhost.to", 10) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -26,11 +24,11 @@ internal class PixhostHost : Host("pixhost.to", 10) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl)
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -12,12 +11,11 @@ internal class PixxxelsHost : Host("pixxxels.cc", 12) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -25,11 +23,11 @@ internal class PixxxelsHost : Host("pixxxels.cc", 12) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
val titleNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, TITLE_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -37,11 +35,11 @@ internal class PixxxelsHost : Host("pixxxels.cc", 12) {
String.format(
"Xpath '%s' cannot be found in '%s'",
TITLE_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = titleNode.textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' }
Pair(
@@ -1,60 +1,33 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
import java.util.*
internal class PostImgHost : Host("postimg.cc", 13) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val titleNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.url))
XpathUtils.getAsNode(document, TITLE_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'",
TITLE_XPATH,
image.url
)
)
val urlNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
val document = fetchDocument(context.imageEntity.url.replace("http:", "https:"), context)
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
val node = XpathUtils.getAsNode(document, IMG_XPATH) ?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
val imgTitle = Optional.ofNullable(titleNode)
.map { node: Node -> node.textContent.trim { it <= ' ' } }
.orElseGet { getDefaultImageName(image.url) }
Pair(imgTitle, urlNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' })
} catch (e: Exception) {
throw HostException("Unexpected error occurred", e)
}
return Pair(
node.attributes.getNamedItem("alt").textContent.trim(),
node.attributes.getNamedItem("src").textContent.trim()
)
}
companion object {
private const val TITLE_XPATH = "//span[contains(@class,'imagename')]"
private const val IMG_XPATH = "//a[@id='download']"
private const val IMG_XPATH = "//img[contains(@class,'img-fluid')]"
}
}
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -13,28 +12,27 @@ internal class TurboImageHost : Host("turboimagehost.com", 14) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
var title: String?
title = try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, context.imageEntity.url))
val titleNode: Node? = XpathUtils.getAsNode(document, TITLE_XPATH)
log.debug(String.format("Resolving name for %s", image.url))
log.debug(String.format("Resolving name for %s", context.imageEntity.url))
titleNode?.textContent?.trim { it <= ' ' }
} catch (e: XpathException) {
throw HostException(e)
}
if (title.isNullOrEmpty()) {
title = getDefaultImageName(image.url)
title = getDefaultImageName(context.imageEntity.url)
}
val urlNode: Node = XpathUtils.getAsNode(document, IMG_XPATH)
?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return Pair(title, urlNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
@@ -1,9 +1,8 @@
package me.vripper.host
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Node
@@ -14,12 +13,11 @@ internal class ViprImHost : Host("vipr.im", 15) {
@Throws(HostException::class)
override fun resolve(
image: ImageEntity,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(image.url, context)
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -27,16 +25,17 @@ internal class ViprImHost : Host("vipr.im", 15) {
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
image.url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", image.url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle =
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { obj: Node -> obj.textContent }
.map { obj: String -> obj.trim() }.orElse(null)
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim()
context.headers["Referer"] = "https://vipr.im/"
Pair(imgTitle!!, imgUrl)
} catch (e: Exception) {
throw HostException("Unexpected error occurred", e)
@@ -5,12 +5,13 @@ import kotlinx.coroutines.flow.filterIsInstance
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.services.*
import me.vripper.services.download.DownloadService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
object AppManager : KoinComponent {
private val eventBus: EventBus by inject()
private val dataTransaction: DataTransaction by inject()
private val dataAccessService: DataAccessService by inject()
private val metadataService: MetadataService by inject()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
@@ -37,8 +38,8 @@ object AppManager : KoinComponent {
threadCacheService.invalidate()
}
}
dataTransaction.setDownloadingToStopped()
dataTransaction.stopImagesByPostIdAndIsNotCompleted()
dataAccessService.setDownloadingToStopped()
dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted()
settingsService.init()
metadataService.fetchExisting()
downloadSpeedService.init()
@@ -20,11 +20,10 @@ data class Post(
val total: Int,
val hosts: Set<String>,
val downloadDirectory: String,
@Contextual val addedOn: LocalDateTime = LocalDateTime.now(),
@Contextual val addedOn: LocalDateTime,
var folderName: String,
var status: Status = Status.STOPPED,
var done: Int = 0,
var rank: Int = Int.MAX_VALUE,
var size: Long = -1,
var downloaded: Long = 0,
val previews: List<String>,
@@ -0,0 +1,3 @@
package me.vripper.model
data class PostIdentifier(val siteProxy: String, val threadId: Long, val postId: Long)
@@ -5,5 +5,6 @@ import kotlinx.serialization.Serializable
@Serializable
data class QueueState(
val running: Int,
val remaining: Int
val remaining: Int,
val rank: List<Rank> = emptyList()
)
@@ -0,0 +1,6 @@
package me.vripper.model
import kotlinx.serialization.Serializable
@Serializable
data class Rank(val postEntityId: Long, val rank: Long)
@@ -5,12 +5,27 @@ import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import java.nio.file.Files
import kotlin.io.path.pathString
enum class HostName {
IMX
}
enum class HostSettingKey(val type: SettingType) {
TRY_TO_FETCH_ORIGINAL_FILENAME(SettingType.BOOLEAN)
}
enum class SettingType {
STRING, BOOLEAN, INT
}
@Serializable
data class Settings(
val connectionSettings: ConnectionSettings = ConnectionSettings(),
val downloadSettings: DownloadSettings = DownloadSettings(),
val viperSettings: ViperSettings = ViperSettings(),
val systemSettings: SystemSettings = SystemSettings()
val systemSettings: SystemSettings = SystemSettings(),
val hostSettings: Map<HostName, Map<HostSettingKey, String>> = mapOf(
HostName.IMX to mapOf(HostSettingKey.TRY_TO_FETCH_ORIGINAL_FILENAME to "false")
)
)
@Serializable
@@ -51,3 +66,4 @@ data class SystemSettings(
val clipboardPollingRate: Int = 500,
val maxEventLog: Int = 1_000,
)
@@ -1,5 +1,7 @@
package me.vripper.services
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.FlowPreview
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.time.sample
import kotlinx.serialization.json.Json
@@ -7,13 +9,13 @@ import me.vripper.entities.*
import me.vripper.event.*
import me.vripper.exception.PostParseException
import me.vripper.model.*
import me.vripper.services.download.DownloadService
import me.vripper.services.download.MovePosition
import me.vripper.services.download.QueueManager
import me.vripper.tasks.AddPostTask
import me.vripper.tasks.ThreadLookupTask
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.*
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.taskRunner
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
import java.sql.DriverManager
import java.time.Duration
@@ -24,9 +26,11 @@ import kotlin.io.path.Path
import kotlin.io.path.exists
import kotlin.jvm.optionals.getOrNull
@OptIn(ExperimentalCoroutinesApi::class, FlowPreview::class)
internal class AppEndpointService(
private val downloadService: DownloadService,
private val dataTransaction: DataTransaction,
private val queueManager: QueueManager,
private val dataAccessService: DataAccessService,
private val threadCacheService: ThreadCacheService,
private val settingsService: SettingsService,
private val vgAuthService: VGAuthService,
@@ -43,15 +47,17 @@ internal class AppEndpointService(
}
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isBlank() }.map { it.trim() }
.filter { it.isNotEmpty() }
val proxies = settingsService.getProxies()
for (link in urlList) {
log.debug("Scanning: $link")
if (!link.startsWith(settingsService.settings.viperSettings.host)) {
val matchingProxy = proxies.find { link.startsWith(it) }
if (matchingProxy == null) {
continue
}
var threadId: Long
var postId: Long?
val m = Pattern.compile(
Pattern.quote(settingsService.settings.viperSettings.host) + "/threads/(\\d+)((.*p=)(\\d+))?"
Pattern.quote(matchingProxy) + "/threads/(\\d+)((.*p=)(\\d+))?"
).matcher(link)
if (m.find()) {
threadId = m.group(1).toLong()
@@ -59,13 +65,13 @@ internal class AppEndpointService(
if (postId == null) {
taskRunner.submit(
ThreadLookupTask(
threadId, settingsService.settings
matchingProxy, threadId, settingsService.settings
)
)
} else {
taskRunner.submit(
AddPostTask(
listOf(ThreadPostId(threadId, postId))
listOf(PostIdentifier(matchingProxy, threadId, postId))
)
)
}
@@ -77,40 +83,46 @@ internal class AppEndpointService(
}
}
override suspend fun restartAll(posIds: List<Long>) {
override suspend fun restartAll(postEntityIds: List<Long>) {
lock.withLock {
downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
.map { dataTransaction.findPostByPostId(it) })
downloadService.restartAll(postEntityIds.filter { dataAccessService.exists(it) }
.map { dataAccessService.findPostByEntityId(it) })
}
}
override suspend fun download(posts: List<ThreadPostId>) {
taskRunner.submit(AddPostTask(posts))
}
override suspend fun stopAll(postIdList: List<Long>) {
lock.withLock {
downloadService.stop(postIdList)
posts.map {
val thread = dataAccessService.findThreadByThreadId(it.threadId).getOrNull()
?: throw PostParseException("Could not find thread with id ${it.threadId}")
PostIdentifier(thread.link.extractBaseUrl(), it.threadId, it.postId)
}.also {
taskRunner.submit(AddPostTask(it))
}
}
override suspend fun remove(postIdList: List<Long>) {
override suspend fun stopAll(postEntityIds: List<Long>) {
lock.withLock {
downloadService.stop(postIdList)
dataTransaction.removeAll(postIdList)
downloadService.stop(postEntityIds)
}
}
override suspend fun remove(postEntityIds: List<Long>) {
lock.withLock {
downloadService.stop(postEntityIds)
dataAccessService.removeAll(postEntityIds)
}
}
override suspend fun clearCompleted(): List<Long> {
lock.withLock {
return dataTransaction.clearCompleted()
return dataAccessService.clearCompleted()
}
}
override suspend fun grab(threadId: Long): List<PostSelection> {
lock.withLock {
return try {
val thread = dataTransaction.findThreadByThreadId(threadId).orElseThrow {
val thread = dataAccessService.findThreadByThreadId(threadId).orElseThrow {
PostParseException(
String.format(
"Unable to find links for threadId = %s", threadId
@@ -148,42 +160,42 @@ internal class AppEndpointService(
override suspend fun threadRemove(threadIdList: List<Long>) {
lock.withLock {
threadIdList.forEach {
dataTransaction.removeThread(it)
dataAccessService.removeThread(it)
}
}
}
override suspend fun threadClear() {
lock.withLock {
dataTransaction.clearQueueLinks()
dataAccessService.clearQueueLinks()
}
}
override suspend fun renameToFirst(postIds: List<Long>) {
postIds.forEach { postId ->
dataTransaction
.findMetadataByPostId(postId)
override suspend fun renameToFirst(postEntityIds: List<Long>) {
postEntityIds.forEach { postEntityId ->
dataAccessService
.findMetadataByPostEntityId(postEntityId)
.map { it.data.resolvedNames }
.filter { it.isNotEmpty() }
.getOrNull()?.let { rename(postId, it.first()) }
.getOrNull()?.let { rename(postEntityId, it.first()) }
}
}
override suspend fun rename(postId: Long, newName: String) {
override suspend fun rename(postEntityId: Long, newName: String) {
taskRunner.submit {
synchronized(postId.toString().intern()) {
if (dataTransaction.exists(postId)) {
dataTransaction.findPostByPostId(postId).let { post ->
synchronized(postEntityId.toString().intern()) {
if (dataAccessService.exists(postEntityId)) {
dataAccessService.findPostByEntityId(postEntityId).let { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId),
dataAccessService.findImagesByPostEntityId(postEntityId),
post.downloadDirectory,
post.folderName,
newName
)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
dataAccessService.updatePost(post)
}
}
}
@@ -201,7 +213,7 @@ internal class AppEndpointService(
override fun onDeletePosts() =
EventBus.events.filterIsInstance(PostDeleteEvent::class).flatMapConcat { it.postIds.asFlow() }
EventBus.events.filterIsInstance(PostDeleteEvent::class).flatMapConcat { it.postEntityIds.asFlow() }
override fun onUpdateMetadata() =
@@ -209,12 +221,12 @@ internal class AppEndpointService(
override suspend fun findAllPosts(): List<Post> {
return dataTransaction.findAllPosts().map(::mapper)
return dataAccessService.findAllPosts().map(::mapper)
}
private fun mapper(postEntity: PostEntity): Post {
val metadata: Metadata? = dataTransaction.findMetadataByPostId(postEntity.postId).orElse(null)
val images = dataTransaction.findImagesByPostId(postEntity.postId)
val metadata: Metadata? = dataAccessService.findMetadataByPostEntityId(postEntity.id).orElse(null)
val images = dataAccessService.findImagesByPostEntityId(postEntity.id)
return Post(
postEntity.id,
postEntity.postTitle,
@@ -222,8 +234,8 @@ internal class AppEndpointService(
postEntity.forum,
postEntity.url,
postEntity.token,
postEntity.postId,
postEntity.threadId,
postEntity.vgPostId,
postEntity.vgThreadId,
postEntity.total,
postEntity.hosts,
postEntity.downloadDirectory,
@@ -231,7 +243,6 @@ internal class AppEndpointService(
postEntity.folderName,
postEntity.status,
postEntity.done,
postEntity.rank,
postEntity.size,
postEntity.downloaded,
images.take(4).map { it.thumbUrl },
@@ -240,17 +251,17 @@ internal class AppEndpointService(
)
}
override suspend fun findPost(postId: Long): Post {
return mapper(dataTransaction.findPostByPostId(postId))
override suspend fun findPost(postEntityId: Long): Post {
return mapper(dataAccessService.findPostByEntityId(postEntityId))
}
override suspend fun findImagesByPostId(postId: Long): List<Image> {
return dataTransaction.findImagesByPostId(postId)
override suspend fun findImagesByPostEntityId(postEntityId: Long): List<Image> {
return dataAccessService.findImagesByPostEntityId(postEntityId)
}
override fun onUpdateImagesByPostId(postId: Long): Flow<Image> =
override fun onUpdateImagesByPostEntityId(postEntityId: Long): Flow<Image> =
EventBus.events.filterIsInstance(ImageEvent::class).map {
it.imageEntities.filter { imageEntity: Image -> imageEntity.postId == postId }
it.imageEntities.filter { imageEntity: Image -> imageEntity.postEntityId == postEntityId }
}.filter { it.isNotEmpty() }.flatMapConcat { it.asFlow() }
override fun onUpdateImages(): Flow<Image> =
@@ -279,7 +290,7 @@ internal class AppEndpointService(
override suspend fun findAllThreads(): List<Thread> {
return dataTransaction.findAllThreads()
return dataAccessService.findAllThreads()
}
override fun onDownloadSpeed(): Flow<DownloadSpeed> =
@@ -291,6 +302,10 @@ internal class AppEndpointService(
override fun onQueueStateUpdate(): Flow<QueueState> =
EventBus.events.filterIsInstance(QueueStateEvent::class).map { it.queueState }
override suspend fun getQueueState(): QueueState {
return queueManager.getQueueState()
}
override fun onErrorCountUpdate(): Flow<ErrorCount> =
EventBus.events.filterIsInstance(ErrorCountEvent::class).map { it.errorCount }
@@ -310,6 +325,10 @@ internal class AppEndpointService(
override suspend fun getVersion(): String = ApplicationProperties.VERSION
override suspend fun move(postEntityId: Long, position: MovePosition) {
downloadService.move(postEntityId, position)
}
override suspend fun dbMigration(): String {
val conn = try {
@@ -319,31 +338,31 @@ internal class AppEndpointService(
return "Old database not found, nothing to do"
}
var postsCount = 0;
var threadCount = 0;
var postsCount = 0
var threadCount = 0
conn.use { conn ->
conn.prepareStatement("select * from post").use {
it.executeQuery().use {
while (it.next()) {
val done = it.getInt("DONE")
val hosts = it.getString("HOSTS")
val outputPath = it.getString("OUTPUT_PATH")
val postId = it.getLong("POST_ID")
val status = it.getString("STATUS")
val threadId = it.getLong("THREAD_ID")
val postTitle = it.getString("POST_TITLE")
val threadTitle = it.getString("THREAD_TITLE")
val forum = it.getString("FORUM")
val total = it.getInt("TOTAL")
val size = it.getLong("SIZE")
val downloaded = it.getLong("DOWNLOADED")
val url = it.getString("URL")
val token = it.getString("TOKEN")
val addedAt = it.getTimestamp("ADDED_AT")
val folderName = it.getString("FOLDER_NAME") ?: ""
conn.prepareStatement("select * from post").use { statement ->
statement.executeQuery().use { resultSet ->
while (resultSet.next()) {
val done = resultSet.getInt("DONE")
val hosts = resultSet.getString("HOSTS")
val outputPath = resultSet.getString("OUTPUT_PATH")
val postId = resultSet.getLong("POST_ID")
val status = resultSet.getString("STATUS")
val threadId = resultSet.getLong("THREAD_ID")
val postTitle = resultSet.getString("POST_TITLE")
val threadTitle = resultSet.getString("THREAD_TITLE")
val forum = resultSet.getString("FORUM")
val total = resultSet.getInt("TOTAL")
val size = resultSet.getLong("SIZE")
val downloaded = resultSet.getLong("DOWNLOADED")
val url = resultSet.getString("URL")
val token = resultSet.getString("TOKEN")
val addedAt = resultSet.getTimestamp("ADDED_AT")
val folderName = resultSet.getString("FOLDER_NAME") ?: ""
val exists = dataTransaction.exists(postId)
val exists = dataAccessService.existsPostId(postId)
if (exists) {
continue
}
@@ -354,8 +373,8 @@ internal class AppEndpointService(
forum = forum,
url = url,
token = token,
postId = postId,
threadId = threadId,
vgPostId = postId,
vgThreadId = threadId,
total = total,
hosts = hosts.split(";").dropLastWhile { it.isEmpty() }.toSet(),
downloadDirectory = outputPath,
@@ -384,7 +403,6 @@ internal class AppEndpointService(
val fileName = set.getString("FILENAME") ?: ""
ImageEntity(
postId = postId,
url = url,
thumbUrl = thumbUrl,
host = host,
@@ -398,7 +416,7 @@ internal class AppEndpointService(
}
}
dataTransaction.saveAndNotify(post, images)
val savedPost = dataAccessService.saveAndNotify(post, images)
//load meta
conn.prepareStatement("select * from metadata where post_id = ?").use { statement ->
@@ -407,10 +425,10 @@ internal class AppEndpointService(
if (set.next()) {
val data = Json.decodeFromString(set.getString("DATA")) as MetadataEntity.Data
val metadata = MetadataEntity(
postId = postId,
postIdRef = savedPost.id,
data = data
)
dataTransaction.saveMetadata(metadata)
dataAccessService.saveMetadata(metadata)
}
}
}
@@ -427,7 +445,7 @@ internal class AppEndpointService(
val threadId = set.getLong("THREAD_ID")
val title = set.getString("TITLE")
if (dataTransaction.findThreadByThreadId(threadId).isPresent) {
if (dataAccessService.findThreadByThreadId(threadId).isPresent) {
continue
}
@@ -438,7 +456,7 @@ internal class AppEndpointService(
total = total,
)
dataTransaction.save(threadEntity)
dataAccessService.save(threadEntity)
threadCount++
}
}
@@ -9,16 +9,16 @@ import me.vripper.data.repositories.ThreadRepository
import me.vripper.entities.*
import me.vripper.event.*
import me.vripper.model.ErrorCount
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.PathUtils.sanitize
import me.vripper.vgapi.PostItem
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlin.io.path.pathString
internal class DataTransaction(
internal class DataAccessService(
private val settingsService: SettingsService,
private val postRepository: PostRepository,
private val imageRepository: ImageRepository,
@@ -27,59 +27,62 @@ internal class DataTransaction(
private val eventBus: EventBus,
) {
private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
private val log by LoggerDelegate()
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findById(id) }
}
private val postPostIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findByPostId(id) }
}
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
return transaction { postRepository.save(postEntities) }
}
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>): PostEntity {
val savedPost = transaction {
val savedPost =
postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
save(images.map { it.copy(postIdRef = savedPost.id) })
postRepository.save(listOf(postEntity)).first()
save(images.map { it.copy(postEntityId = savedPost.id) })
// Publish event inside transaction for consistency
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
savedPost
}
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
return savedPost
}
fun updatePosts(postEntities: List<PostEntity>) {
transaction { postRepository.update(postEntities) }
postEntities.forEach { postEntity ->
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
}
// Log and publish event after transaction commit but before returning
log.debug("[{}] Publishing event: PostUpdateEvent for {} posts", System.currentTimeMillis(), postEntities.size)
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
fun updatePost(postEntity: PostEntity) {
transaction { postRepository.update(postEntity) }
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
// Log and publish event after transaction commit but before returning
log.debug("[{}] Publishing event: PostUpdateEvent for post {}", System.currentTimeMillis(), postEntity.id)
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
fun save(threadEntity: ThreadEntity) {
val savedThread = transaction { threadRepository.save(threadEntity) }
log.debug("[{}] Publishing event: ThreadCreateEvent for thread {}", System.currentTimeMillis(), savedThread.id)
eventBus.publishEvent(ThreadCreateEvent(savedThread))
}
fun update(threadEntity: ThreadEntity) {
transaction { threadRepository.update(threadEntity) }
log.debug("[{}] Publishing event: ThreadUpdateEvent for thread {}", System.currentTimeMillis(), threadEntity.id)
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
}
fun updateImages(imageEntities: List<ImageEntity>) {
transaction { imageRepository.update(imageEntities) }
log.debug("[{}] Publishing event: ImageEvent for {} images", System.currentTimeMillis(), imageEntities.size)
eventBus.publishEvent(ImageEvent(imageEntities))
}
@@ -89,13 +92,18 @@ internal class DataTransaction(
imageRepository.update(imageEntity)
}
}
log.debug("[{}] Publishing event: ImageEvent for image {}", System.currentTimeMillis(), imageEntity.id)
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
}
fun exists(postId: Long): Boolean {
if (postPostIdCache.getIfPresent(postId) != null) {
fun exists(postEntityId: Long): Boolean {
if (postEntityIdCache.getIfPresent(postEntityId) != null) {
return true
}
return transaction { postRepository.existByPostEntityId(postEntityId) }
}
fun existsPostId(postId: Long): Boolean {
return transaction { postRepository.existByPostId(postId) }
}
@@ -106,13 +114,12 @@ internal class DataTransaction(
postTitle = postItem.title,
url = postItem.url,
token = postItem.securityToken,
postId = postItem.postId,
threadId = postItem.threadId,
vgPostId = postItem.postId,
vgThreadId = postItem.threadId,
total = postItem.imageCount,
hosts = postItem.hosts.map { "${it.first} (${it.second})" }.toSet(),
threadTitle = postItem.threadTitle,
forum = postItem.forum,
rank = nextRank.andIncrement,
downloadDirectory = PathUtils.calculateDownloadPath(
postItem.forum,
postItem.threadTitle,
@@ -124,7 +131,6 @@ internal class DataTransaction(
)
val imageEntities = postItem.imageItemList.mapIndexed { index, imageItem ->
ImageEntity(
postId = postItem.postId,
url = imageItem.mainLink,
thumbUrl = imageItem.thumbLink,
host = imageItem.host.hostId,
@@ -137,10 +143,10 @@ internal class DataTransaction(
val savedPosts = transaction {
val savedPosts = save(posts.keys.toList())
savedPosts.associateWith {
posts[it]!!
savedPosts.associateWith { postEntity ->
posts.entries.find { postItem -> postItem.key.vgPostId == postEntity.vgPostId }?.value ?: emptyList()
}.forEach { (key, value) ->
save(value.map { it.copy(postIdRef = key.id) })
save(value.map { it.copy(postEntityId = key.id) })
}
savedPosts
}
@@ -148,17 +154,13 @@ internal class DataTransaction(
return savedPosts
}
private fun getQueuePosition(): Int? {
return postRepository.findMaxRank()
}
private fun save(imageEntities: List<ImageEntity>) {
transaction { imageRepository.save(imageEntities) }
}
fun finishPost(postId: Long, automatic: Boolean = false) {
val post = findPostByPostId(postId)
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
fun finishPost(id: Long, automatic: Boolean = false) {
val post = findPostByEntityId(id)
val imagesInErrorStatus = findByIdAndIsError(id)
if (imagesInErrorStatus.isNotEmpty()) {
post.status = Status.ERROR
updatePost(post)
@@ -171,36 +173,37 @@ internal class DataTransaction(
transaction {
updatePost(post)
if (settingsService.settings.downloadSettings.clearCompleted && automatic) {
remove(listOf(post.postId))
remove(listOf(post.id))
}
}
}
}
}
private fun findByPostIdAndIsError(postId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostIdAndIsError(postId) }
private fun findByIdAndIsError(postEntityId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostEntityIdAndIsError(postEntityId) }
}
private fun remove(postIds: List<Long>) {
private fun remove(postEntityIds: List<Long>) {
transaction {
metadataRepository.deleteAllByPostId(postIds)
imageRepository.deleteAllByPostId(postIds)
postRepository.deleteAll(postIds)
sortPostsByRank()
metadataRepository.deleteAllByPostEntityId(postEntityIds)
imageRepository.deleteAllByPostEntityId(postEntityIds)
postRepository.deleteAll(postEntityIds)
}
postIds.forEach { postId ->
postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
postPostIdCache.invalidate(postId)
postEntityIds.forEach { postEntityId ->
postEntityIdCache.get(postEntityId)?.let { postEntityIdCache.invalidate(it.id) }
postEntityIdCache.invalidate(postEntityId)
}
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
log.debug("[{}] Publishing event: PostDeleteEvent for {} posts", System.currentTimeMillis(), postEntityIds.size)
eventBus.publishEvent(PostDeleteEvent(postEntityIds = postEntityIds))
log.debug("[{}] Publishing event: ErrorCountEvent after post deletion", System.currentTimeMillis())
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
fun removeThread(threadId: Long) {
transaction { threadRepository.deleteByThreadId(threadId) }
log.debug("[{}] Publishing event: ThreadDeleteEvent for thread {}", System.currentTimeMillis(), threadId)
eventBus.publishEvent(ThreadDeleteEvent(threadId))
}
@@ -210,45 +213,38 @@ internal class DataTransaction(
return completed
}
fun removeAll(postIds: List<Long> = emptyList()) {
if (postIds.isNotEmpty()) {
remove(postIds)
fun removeAll(postEntityIds: List<Long> = emptyList()) {
if (postEntityIds.isNotEmpty()) {
remove(postEntityIds)
} else {
remove(findAllPosts().map(PostEntity::postId))
remove(findAllPosts().map(PostEntity::id))
}
}
fun stopImagesByPostIdAndIsNotCompleted(postId: Long) {
transaction { imageRepository.stopByPostIdAndIsNotCompleted(postId) }
fun stopImagesByPostEntityIdAndIsNotCompleted(postEntityId: Long) {
transaction { imageRepository.stopByPostEntityIdAndIsNotCompleted(postEntityId) }
}
fun stopImagesByPostIdAndIsNotCompleted() {
transaction { imageRepository.stopByPostIdAndIsNotCompleted() }
fun stopImagesByPostEntityIdAndIsNotCompleted() {
transaction { imageRepository.stopByPostEntityIdAndIsNotCompleted() }
}
fun saveMetadata(metadataEntity: MetadataEntity) {
transaction { metadataRepository.save(metadataEntity) }
log.debug(
"[{}] Publishing event: MetadataUpdateEvent for post {}",
System.currentTimeMillis(),
metadataEntity.postIdRef
)
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
}
fun clearQueueLinks() {
transaction { threadRepository.deleteAll() }
log.debug("[{}] Publishing event: ThreadClearEvent", System.currentTimeMillis())
eventBus.publishEvent(ThreadClearEvent())
}
@Synchronized
fun sortPostsByRank() {
val postsToUpdate = mutableListOf<PostEntity>()
val postEntities = findAllPosts().sortedWith(Comparator.comparing(PostEntity::rank))
for (i in postEntities.indices) {
if (postEntities[i].rank != i) {
postsToUpdate.add(postEntities[i].copy(rank = i))
}
}
updatePosts(postsToUpdate)
nextRank.set(transaction { getQueuePosition() }?.plus(1) ?: 0)
}
fun setDownloadingToStopped() {
transaction { postRepository.setDownloadingToStopped() }
}
@@ -257,12 +253,12 @@ internal class DataTransaction(
return transaction { postRepository.findAll() }
}
fun findPostById(id: Long): PostEntity {
fun findPostByEntityId(id: Long): PostEntity {
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
}
fun findImagesByPostId(postId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostId(postId) }
fun findImagesByPostEntityId(postEntityId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostEntityId(postEntityId) }
}
fun findImageById(id: Long): Optional<ImageEntity> {
@@ -279,27 +275,23 @@ internal class DataTransaction(
}
}
fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostIdAndIsNotCompleted(postId) }
fun findByPostEntityIdAndIsNotCompleted(postEntityId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostEntityIdAndIsNotCompleted(postEntityId) }
}
fun countImagesInError(): Int {
return transaction { imageRepository.countError() }
}
fun findPostByPostId(postId: Long): PostEntity {
return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
}
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
return transaction { threadRepository.findByThreadId(threadId) }
}
fun findAllNonCompletedPostIds(): List<Long> {
return transaction { postRepository.findAllNonCompletedPostIds() }
fun findAllNonCompletedPostEntityIds(): List<Long> {
return transaction { postRepository.findAllNonCompletedPostEntityIds() }
}
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
return transaction { metadataRepository.findByPostId(postId) }
fun findMetadataByPostEntityId(postEntityId: Long): Optional<MetadataEntity> {
return transaction { metadataRepository.findByPostEntityId(postEntityId) }
}
}
@@ -1,499 +0,0 @@
package me.vripper.services
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import dev.failsafe.function.CheckedRunnable
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
import me.vripper.event.ErrorCountEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.event.StoppedEvent
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.ErrorCount
import me.vripper.model.QueueState
import me.vripper.model.Settings
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import me.vripper.utilities.downloadRunner
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
import org.apache.hc.client5.http.cookie.Cookie
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.jetbrains.exposed.sql.transactions.transaction
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.time.Instant
import java.util.*
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
import kotlin.io.path.Path
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.pathString
import kotlin.io.path.readLines
internal class DownloadService(
private val settingsService: SettingsService,
private val dataTransaction: DataTransaction,
private val retryPolicyService: RetryPolicyService,
private val eventBus: EventBus
) {
private val maxPoolSize: Int = 24
private val log by LoggerDelegate()
private val running: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
private val lock = ReentrantLock()
private val condition = lock.newCondition()
private var downloadMonitorThread: Thread? = null
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
private val log by LoggerDelegate()
init {
ApplicationProperties.VRIPPER_DIR
.listDirectoryEntries()
.filter { it.fileName.pathString.startsWith("cookies") }
.forEach { cookiesPath ->
loadCookies(cookiesPath).also { cookies ->
cookies.forEach { cookie ->
if (HTTPService.cookieStore.cookies.find { it.name == cookie.name } == null) {
log.info("Applying cookie: ${cookie.name}")
HTTPService.cookieStore.addCookie(cookie)
} else {
log.warn("Cookie already loaded: ${cookie.name}")
}
}
}
}
}
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext = HttpClientContext.create().apply {
cookieStore = HTTPService.cookieStore
}
private fun loadCookies(cookiesPath: Path): List<Cookie> {
return cookiesPath.readLines()
.map { it.trim() }
.filter { it.isNotBlank() }
.filter { !it.startsWith("#") }
.map { line ->
val cookieComponents = line.split("\t")
BasicClientCookie(cookieComponents[5], cookieComponents[6]).apply {
domain = cookieComponents[0]
isHttpOnly = cookieComponents[1].toBoolean()
path = cookieComponents[2]
isSecure = cookieComponents[3].toBoolean()
setExpiryDate(Instant.ofEpochSecond(cookieComponents[4].toLong()))
}
}.also {
log.info("Found ${it.size} cookies in $cookiesPath")
}
}
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
fun cancelCoroutines() {
runBlocking {
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
}
internal class ImageDownloadRunnable(
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val vgauthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
private lateinit var context: ImageDownloadContext
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
vgauthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
val host = hosts.first { it.isSupported(imageEntity.url) }
val downloadedImage = host.downloadInternal(imageEntity, context)
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
)
if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
imageEntity.status = Status.FINISHED
post.done += 1
post.downloaded += imageEntity.size
dataTransaction.updatePost(post)
} else {
imageEntity.status = Status.ERROR
}
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (stopped) {
return
}
imageEntity.status = Status.ERROR
dataTransaction.updateImage(imageEntity)
throw DownloadException(e)
}
}
@Throws(HostException::class)
private fun checkImageTypeAndRename(
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
) {
val existingExtension = getExtension(downloadedImage.name).lowercase()
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
val extension = when (downloadedImage.type) {
ImageMimeType.IMAGE_BMP -> "BMP"
ImageMimeType.IMAGE_GIF -> "GIF"
ImageMimeType.IMAGE_JPEG -> "JPG"
ImageMimeType.IMAGE_PNG -> "PNG"
ImageMimeType.IMAGE_WEBP -> "WEBP"
}
val filename =
if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
sanitize(
fileNameWithoutExtension
)
}.$extension"
try {
val downloadDestinationFolder = Path.of(downloadDirectory)
Files.createDirectories(downloadDestinationFolder)
val finalFilename = "${
if (settings.downloadSettings.forceOrder) String.format(
"%03d_", index + 1
) else ""
}$filename"
imageEntity.filename = finalFilename
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
} catch (e: Exception) {
throw HostException("Failed to rename the image", e)
} finally {
try {
Files.delete(downloadedImage.path)
} catch (_: IOException) {
}
}
}
override fun run() {
context = ImageDownloadContext(imageEntity, settings)
try {
if (stopped) {
return
}
download()
} finally {
completed = true
context.cancelCoroutines()
}
}
fun stop() {
stopped = true
context.requests.forEach { it.abort() }
context.cancelCoroutines()
dataTransaction.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
val that = other as ImageDownloadRunnable
return imageEntity.id == that.imageEntity.id
}
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
}
fun init() {
downloadMonitorThread?.interrupt()
downloadMonitorThread = Thread.ofVirtual().name("Download Monitor").unstarted(Runnable {
log.info("Scheduler have been initialized")
val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
while (!Thread.currentThread().isInterrupted) {
lock.withLock {
candidates.addAll(getCandidates(candidateCount()))
candidates.forEach {
if (canRun(it.imageEntity.host)) {
accepted.add(it)
running[it.imageEntity.host]!!.add(it)
log.debug("${it.imageEntity.url} accepted to run")
}
}
accepted.forEach {
pending[it.imageEntity.host]?.remove(it)
scheduleForDownload(it)
}
accepted.clear()
candidates.clear()
try {
condition.await()
} catch (_: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
log.info("Scheduler have been shutdown")
})
downloadMonitorThread?.start()
}
fun halt() {
downloadMonitorThread?.interrupt()
}
fun stop(postIds: List<Long> = emptyList()) {
if (postIds.isNotEmpty()) {
stopInternal(postIds)
eventBus.publishEvent(StoppedEvent(postIds))
} else {
stopAll()
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
}
fun restartAll(postEntityIds: List<PostEntity> = emptyList()) {
if (postEntityIds.isNotEmpty()) {
restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
} else {
restart(
dataTransaction.findAllPosts()
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
}
}
private fun restart(posts: Map<PostEntity, List<ImageEntity>>) {
lock.withLock {
val toProcess = mutableMapOf<PostEntity, List<ImageEntity>>()
for ((post, images) in posts) {
if (images.isEmpty()) {
continue
}
if (isPending(post.postId)) {
continue
}
toProcess[post] = images
}
toProcess.forEach { (post, images) ->
post.status = Status.PENDING
images.forEach { image ->
with(image) {
this.status = Status.PENDING
this.downloaded = 0
}
}
}
transaction {
dataTransaction.updatePosts(
toProcess.keys.toList()
)
dataTransaction.updateImages(
toProcess.values.flatten()
)
}
toProcess.entries.forEach { (post, images) ->
images.forEach { image ->
log.debug("Enqueuing a job for ${image.url}")
val imageDownloadRunnable = ImageDownloadRunnable(
image, post.rank, settingsService.settings
)
pending.computeIfAbsent(
image.host
) { mutableListOf() }
pending[image.host]!!.add(imageDownloadRunnable)
}
}
condition.signal()
}
}
private fun isPending(postId: Long): Boolean {
return pending.values.flatten().any { it.imageEntity.postId == postId }
}
private fun isRunning(postId: Long): Boolean {
return running.values.flatten().any { it.imageEntity.postId == postId }
}
private fun stopAll() {
lock.withLock {
pending.values.clear()
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten().count { !it.completed } > 0) {
Thread.sleep(100)
}
dataTransaction.findAllNonCompletedPostIds().forEach {
dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
dataTransaction.finishPost(it)
}
}
}
private fun stopInternal(postIds: List<Long>) {
lock.withLock {
for (postId in postIds) {
pending.values.forEach { pending ->
pending.removeIf { it.imageEntity.postId == postId }
}
running.values.flatten()
.filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten()
.count { !it.completed && it.imageEntity.postId == postId } > 0
) {
Thread.sleep(100)
}
}
postIds.forEach {
dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
dataTransaction.finishPost(it)
}
}
}
private fun canRun(host: Byte): Boolean {
val totalRunning = running.values.sumOf { it.size }
return (running[host]!!.size < settingsService.settings.connectionSettings.maxConcurrentPerHost && if (settingsService.settings.connectionSettings.maxGlobalConcurrent == 0) totalRunning < maxPoolSize else totalRunning < settingsService.settings.connectionSettings.maxGlobalConcurrent)
}
private fun candidateCount(): Map<Byte, Int> {
val map: MutableMap<Byte, Int> = mutableMapOf()
Host.Companion.getHosts().values.forEach { host: Byte ->
val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
host
) { mutableListOf() }
val count: Int =
settingsService.settings.connectionSettings.maxConcurrentPerHost - imageDownloadRunnableList.size
log.debug("Download slots for $host: $count")
map[host] = count
}
return map
}
private fun getCandidates(candidateCount: Map<Byte, Int>): List<ImageDownloadRunnable> {
val hostIntegerMap: MutableMap<Byte, Int> = candidateCount.toMutableMap()
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
hosts@ for (host in pending.keys) {
val list: List<ImageDownloadRunnable> =
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
.thenComparingInt { it.imageEntity.index })
for (imageDownloadRunnable in list) {
val count = hostIntegerMap[host] ?: 0
if (count > 0) {
candidates.add(imageDownloadRunnable)
hostIntegerMap[host] = count - 1
} else {
continue@hosts
}
}
}
return candidates
}
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
.with(downloadRunner)
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
it.exception
)
val image = imageDownloadRunnable.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}
.onComplete {
afterJobFinish(imageDownloadRunnable)
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
)
)
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
log.debug(
"Finished downloading ${imageDownloadRunnable.imageEntity.url}"
)
}.runAsync(imageDownloadRunnable)
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
lock.withLock {
val image = imageDownloadRunnable.imageEntity
running[image.host]!!.remove(imageDownloadRunnable)
if (!isPending(image.postId) && !isRunning(
image.postId
) && !imageDownloadRunnable.stopped
) {
dataTransaction.finishPost(image.postId, true)
}
condition.signal()
}
}
private fun pendingCount(): Int {
return pending.values.sumOf { it.size }
}
private fun runningCount(): Int {
return running.values.sumOf { it.size }
}
}
@@ -6,6 +6,7 @@ import me.vripper.event.DownloadSpeedEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.model.DownloadSpeed
import me.vripper.utilities.LoggerDelegate
import java.util.concurrent.atomic.AtomicLong
internal class DownloadSpeedService(
@@ -16,6 +17,7 @@ internal class DownloadSpeedService(
const val DOWNLOAD_POLL_RATE = 2500
}
private val log by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val bytesCount = AtomicLong(0)
private var job: Job? = null
@@ -32,7 +34,13 @@ internal class DownloadSpeedService(
while (isActive) {
delay(DOWNLOAD_POLL_RATE.toLong())
val newValue = bytesCount.getAndSet(0)
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))))
val speed = DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))
log.debug(
"[{}] Publishing event: DownloadSpeedEvent({})",
System.currentTimeMillis(),
speed
)
eventBus.publishEvent(DownloadSpeedEvent(speed))
}
}
}
@@ -40,6 +48,7 @@ internal class DownloadSpeedService(
job?.cancel()
coroutineScope.launch {
delay(DOWNLOAD_POLL_RATE + 500L)
log.debug("[{}] Publishing event: DownloadSpeedEvent(0)", System.currentTimeMillis())
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(0L)))
}
}
@@ -41,7 +41,7 @@ internal class HTTPService {
connectionExpiryJob = coroutineScope.launch {
while (isActive) {
it.closeExpired()
delay(60_000)
delay(300_000)
}
}
}
@@ -2,22 +2,23 @@ package me.vripper.services
import kotlinx.coroutines.flow.Flow
import me.vripper.model.*
import me.vripper.services.download.MovePosition
interface IAppEndpointService {
suspend fun scanLinks(postLinks: String)
suspend fun restartAll(posIds: List<Long> = listOf())
suspend fun remove(postIdList: List<Long>)
suspend fun stopAll(postIdList: List<Long> = emptyList())
suspend fun restartAll(postEntityIds: List<Long> = listOf())
suspend fun remove(postEntityIds: List<Long>)
suspend fun stopAll(postEntityIds: List<Long> = emptyList())
suspend fun clearCompleted(): List<Long>
suspend fun findPost(postId: Long): Post
suspend fun findPost(postEntityId: Long): Post
suspend fun findAllPosts(): List<Post>
suspend fun rename(postId: Long, newName: String)
suspend fun rename(postEntityId: Long, newName: String)
fun onNewPosts(): Flow<Post>
fun onUpdatePosts(): Flow<Post>
fun onDeletePosts(): Flow<Long>
fun onUpdateMetadata(): Flow<Metadata>
suspend fun findImagesByPostId(postId: Long): List<Image>
fun onUpdateImagesByPostId(postId: Long): Flow<Image>
suspend fun findImagesByPostEntityId(postEntityId: Long): List<Image>
fun onUpdateImagesByPostEntityId(postEntityId: Long): Flow<Image>
fun onUpdateImages(): Flow<Image>
fun onStopped(): Flow<Long>
fun onNewLog(): Flow<LogEntry>
@@ -32,6 +33,7 @@ interface IAppEndpointService {
suspend fun download(posts: List<ThreadPostId>)
fun onDownloadSpeed(): Flow<DownloadSpeed>
fun onVGUserUpdate(): Flow<String>
suspend fun getQueueState(): QueueState
fun onQueueStateUpdate(): Flow<QueueState>
fun onErrorCountUpdate(): Flow<ErrorCount>
fun onTasksRunning(): Flow<Boolean>
@@ -41,8 +43,9 @@ interface IAppEndpointService {
fun onUpdateSettings(): Flow<Settings>
suspend fun loggedInUser(): String
suspend fun getVersion(): String
suspend fun renameToFirst(postIds: List<Long>)
suspend fun renameToFirst(postEntityIds: List<Long>)
suspend fun dbMigration(): String
suspend fun initLogger()
suspend fun move(postEntityId: Long, position: MovePosition)
fun connectionState(): String
}
@@ -1,10 +1,11 @@
package me.vripper.services
import me.vripper.entities.PostEntity
import me.vripper.tasks.FetchMetadataTask
import me.vripper.utilities.taskRunner
internal class MetadataService(
private val dataTransaction: DataTransaction,
private val dataAccessService: DataAccessService,
private val settingsService: SettingsService,
) {
@@ -12,17 +13,17 @@ internal class MetadataService(
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
dataTransaction.findAllPosts().filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }
.map { it.postId }.forEach(::fetchMetadata)
dataAccessService.findAllPosts().filter { dataAccessService.findMetadataByPostEntityId(it.id).isEmpty }
.map { it }.forEach(::fetchMetadata)
}
fun fetchMetadata(postId: Long) {
fun fetchMetadata(post: PostEntity) {
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
taskRunner.submit(
FetchMetadataTask(
postId
post
)
)
}
@@ -2,29 +2,21 @@ package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import me.vripper.utilities.extractBaseUrl
import me.vripper.vgapi.ThreadItem
import me.vripper.vgapi.ThreadLookupAPIParser
import java.util.concurrent.ExecutionException
import java.util.concurrent.TimeUnit
import kotlin.jvm.optionals.getOrNull
internal class ThreadCacheService(val dataTransaction: DataTransaction) {
internal class ThreadCacheService(val dataAccessService: DataAccessService) {
private val cache: LoadingCache<Long, ThreadItem> =
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
val threadItem = ThreadLookupAPIParser(threadId).parse()
dataTransaction.findThreadByThreadId(threadItem.threadId).ifPresent {
if (threadItem.postItemList.isNotEmpty()) {
dataTransaction.update(it.copy(total = threadItem.postItemList.size))
}
}
threadItem
}
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build(::cacheLoader)
fun invalidate() {
cache.invalidateAll()
}
@Throws(ExecutionException::class)
operator fun get(threadId: Long): ThreadItem {
return cache[threadId]
}
@@ -32,4 +24,31 @@ internal class ThreadCacheService(val dataTransaction: DataTransaction) {
fun getIfPresent(threadId: Long): ThreadItem? {
return cache.getIfPresent(threadId)
}
fun loadThenCache(threadId: Long, siteProxy: String): ThreadItem {
return cache.get(threadId) {
cacheLoader(threadId, siteProxy)
}
}
private fun cacheLoader(threadId: Long, siteProxy: String? = null): ThreadItem? {
val result = if (siteProxy != null) {
ThreadLookupAPIParser(siteProxy, threadId).parse()
} else {
dataAccessService.findThreadByThreadId(threadId).map { threadEntity ->
ThreadLookupAPIParser(threadEntity.link.extractBaseUrl(), threadId).parse()
}.getOrNull()
}
if (result == null) {
throw Exception("Failed to load thread $threadId")
}
dataAccessService.findThreadByThreadId(result.threadId).ifPresent {
if (result.postItemList.isNotEmpty()) {
dataAccessService.update(it.copy(total = result.postItemList.size))
}
}
return result
}
}
@@ -29,11 +29,14 @@ internal class VGAuthService(
fun authenticate(settings: Settings) {
if (!settings.viperSettings.login) {
log.debug("Authentication option is disabled")
authenticated = false
loggedUser = ""
synchronized(this) {
authenticated = false
loggedUser = ""
}
synchronized(vgCookies) {
vgCookies.clear()
}
log.debug("[{}] Publishing event: VGUserLoginEvent", System.currentTimeMillis())
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
@@ -41,11 +44,14 @@ internal class VGAuthService(
val password = settings.viperSettings.password
if (username.isEmpty() || password.isEmpty()) {
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
authenticated = false
loggedUser = ""
synchronized(this) {
authenticated = false
loggedUser = ""
}
synchronized(vgCookies) {
vgCookies.clear()
}
log.debug("[{}] Publishing event: VGUserLoginEvent (empty credentials)", System.currentTimeMillis())
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
@@ -58,8 +64,9 @@ internal class VGAuthService(
BasicNameValuePair("vb_login_md5password", password)
)
)
it.setAbsoluteRequestUri(true)
}
log.info("Authenticating: ${postAuth.uri}")
log.info("Authenticating: {}", postAuth)
try {
val context = HttpClientContext.create().apply {
cookieStore =
@@ -81,6 +88,7 @@ internal class VGAuthService(
"Failed to authenticate user with {}, missing vg_userid/vg_password cookie",
settings.viperSettings.host
)
log.debug("[${System.currentTimeMillis()}] Publishing event: VGUserLoginEvent (missing cookies)")
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
@@ -97,15 +105,21 @@ internal class VGAuthService(
log.error(
"Failed to authenticate user with " + settings.viperSettings.host, e
)
authenticated = false
loggedUser = ""
synchronized(this) {
authenticated = false
loggedUser = ""
}
log.debug("[{}] Publishing event: VGUserLoginEvent (exception)", System.currentTimeMillis())
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
authenticated = true
loggedUser = username
synchronized(this) {
authenticated = true
loggedUser = username
}
log.debug("[{}] Publishing event: VGUserLoginEvent for user {}", System.currentTimeMillis(), username)
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
log.info("Successfully logged in as: $loggedUser")
log.info("Successfully logged in as: {}", loggedUser)
}
fun leaveThanks(postEntity: PostEntity) {
@@ -0,0 +1,171 @@
package me.vripper.services.download
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
import me.vripper.event.EventBus
import me.vripper.event.StoppedEvent
import me.vripper.services.DataAccessService
import me.vripper.services.SettingsService
import me.vripper.services.download.SharedLock.downloadManagerCondition
import me.vripper.services.download.SharedLock.downloadManagerLock
import me.vripper.utilities.LoggerDelegate
import org.jetbrains.exposed.sql.transactions.transaction
import kotlin.concurrent.withLock
import kotlin.math.min
internal class DownloadService(
private val settingsService: SettingsService,
private val dataAccessService: DataAccessService,
private val queueManager: QueueManager,
private val eventBus: EventBus
) {
private val maxPoolSize: Int = 24
private val log by LoggerDelegate()
private var downloadMonitorThread: Thread? = null
fun init() {
downloadMonitorThread?.interrupt()
downloadMonitorThread = Thread.ofVirtual().name("Download Monitor").unstarted(Runnable {
log.info("DownloadManager have been initialized")
while (!Thread.currentThread().isInterrupted) {
downloadManagerLock.withLock {
val candidates = getCandidates()
val canRunPerHost = canRun(candidates.keys)
val accepted = candidates.entries.flatMap {
it.value.take(canRunPerHost[it.key] ?: 0)
}
queueManager.accept(accepted)
try {
downloadManagerCondition.await()
} catch (_: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
log.info("Scheduler have been shutdown")
})
downloadMonitorThread?.start()
}
fun halt() {
downloadMonitorThread?.interrupt()
}
fun stop(postEntityIds: List<Long> = emptyList()) {
downloadManagerLock.withLock {
if (postEntityIds.isNotEmpty()) {
stopInternal(postEntityIds)
log.debug(
"[{}] Publishing event: StoppedEvent for {} posts",
System.currentTimeMillis(),
postEntityIds.size
)
eventBus.publishEvent(StoppedEvent(postEntityIds))
} else {
stopAll()
log.debug("[{}] Publishing event: StoppedEvent for all posts", System.currentTimeMillis())
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
}
}
fun restartAll(postEntities: List<PostEntity> = emptyList()) {
if (postEntities.isNotEmpty()) {
restart(postEntities.associateWith { dataAccessService.findByPostEntityIdAndIsNotCompleted(it.id) })
} else {
restart(
dataAccessService.findAllPosts()
.associateWith { dataAccessService.findByPostEntityIdAndIsNotCompleted(it.id) })
}
}
fun move(postEntityId: Long, position: MovePosition) {
queueManager.move(postEntityId, position)
}
private fun restart(posts: Map<PostEntity, List<ImageEntity>>) {
downloadManagerLock.withLock {
val toProcess = mutableMapOf<PostEntity, List<ImageEntity>>()
for ((post, images) in posts) {
if (images.isEmpty()) {
continue
}
if (queueManager.isPending(post.id)) {
continue
}
toProcess[post] = images
}
toProcess.forEach { (post, images) ->
post.status = Status.PENDING
images.forEach { image ->
with(image) {
this.status = Status.PENDING
this.downloaded = 0
}
}
}
transaction {
dataAccessService.updatePosts(
toProcess.keys.toList()
)
dataAccessService.updateImages(
toProcess.values.flatten()
)
}
toProcess.entries.sortedBy { it.key.addedOn }.forEach { (post, images) ->
images.map { image ->
log.debug("Enqueuing a job for ${post.url}")
ImageQueueElement(
image.id, image.postEntityId, image.host
)
}.also { queueManager.addPending(it) }
}
downloadManagerCondition.signal()
}
}
private fun stopAll() {
queueManager.clearPending()
queueManager.clearRunning()
dataAccessService.findAllNonCompletedPostEntityIds().forEach {
dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it)
dataAccessService.finishPost(it)
}
}
private fun stopInternal(postEntityIds: List<Long>) {
postEntityIds.forEach {
queueManager.clearPending(it)
queueManager.clearRunning(it)
dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it)
dataAccessService.finishPost(it)
}
}
private fun canRun(hosts: Set<Byte>): Map<Byte, Int> {
val runningSnapshot = queueManager.runningTotalCount().toMutableMap()
return hosts.associateWith {
val totalRunning = runningSnapshot.values.sum()
val canRunGlobally =
if (settingsService.settings.connectionSettings.maxGlobalConcurrent == 0) maxPoolSize - totalRunning else settingsService.settings.connectionSettings.maxGlobalConcurrent - totalRunning
val canRunPerHost =
settingsService.settings.connectionSettings.maxConcurrentPerHost - (runningSnapshot[it] ?: 0)
val min = min(canRunGlobally, canRunPerHost)
val actual = if (min < 0) 0 else min
runningSnapshot[it] = (runningSnapshot[it] ?: 0) + actual
actual
}
}
private fun getCandidates(): Map<Byte, List<ImageQueueElement>> {
return queueManager.pending().groupBy { it.host }
}
}
@@ -0,0 +1,223 @@
package me.vripper.services.download
import dev.failsafe.function.CheckedRunnable
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.entities.Status
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.DataAccessService
import me.vripper.services.HTTPService
import me.vripper.services.VGAuthService
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
import org.apache.hc.client5.http.cookie.Cookie
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.time.Instant
import java.util.*
import kotlin.io.path.Path
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.pathString
import kotlin.io.path.readLines
internal class ImageDownloadRunnable(
imageEntity: ImageEntity, settings: Settings
) : KoinComponent, CheckedRunnable {
class Context(val imageEntity: ImageEntity, val settings: Settings) {
private val log by LoggerDelegate()
private lateinit var coroutineScope: CoroutineScope
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext = HttpClientContext.create().apply {
cookieStore = HTTPService.cookieStore
}
val requests = mutableListOf<HttpUriRequestBase>()
val headers = mutableMapOf<String, String>()
var completed = false
var stopped = false
fun init() {
coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
ApplicationProperties.VRIPPER_DIR.listDirectoryEntries()
.filter { it.fileName.pathString.startsWith("cookies") }.forEach { cookiesPath ->
loadCookies(cookiesPath).also { cookies ->
cookies.forEach { cookie ->
if (HTTPService.cookieStore.cookies.find { it.name == cookie.name } == null) {
log.info("Applying cookie: ${cookie.name}")
HTTPService.cookieStore.addCookie(cookie)
} else {
log.warn("Cookie already loaded: ${cookie.name}")
}
}
}
}
}
private fun loadCookies(cookiesPath: Path): List<Cookie> {
return cookiesPath.readLines().map { it.trim() }.filter { it.isNotBlank() }.filter { !it.startsWith("#") }
.map { line ->
val cookieComponents = line.split("\t")
BasicClientCookie(cookieComponents[5], cookieComponents[6]).apply {
domain = cookieComponents[0]
isHttpOnly = cookieComponents[1].toBoolean()
path = cookieComponents[2]
isSecure = cookieComponents[3].toBoolean()
setExpiryDate(Instant.ofEpochSecond(cookieComponents[4].toLong()))
}
}.also {
log.info("Found ${it.size} cookies in $cookiesPath")
}
}
@Synchronized
fun clear() {
runBlocking {
requests.forEach { it.abort() }
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
jobs.clear()
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
}
private val log by LoggerDelegate()
private val dataAccessService: DataAccessService by inject()
private val vgAuthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
val context = Context(imageEntity, settings)
fun download() {
try {
context.imageEntity.status = Status.DOWNLOADING
context.imageEntity.downloaded = 0
dataAccessService.updateImage(context.imageEntity)
synchronized(context.imageEntity.postEntityId.toString().intern()) {
val post = dataAccessService.findPostByEntityId(context.imageEntity.postEntityId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataAccessService.updatePost(post)
vgAuthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${context.imageEntity.url} using ${context.imageEntity.host}")
val host = hosts.first { it.isSupported(context.imageEntity.url) }
val downloadedImage = host.downloadInternal(context)
log.debug("Resolved name for ${context.imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", context.imageEntity.url, downloadedImage.path)
synchronized(context.imageEntity.postEntityId.toString().intern()) {
val post = dataAccessService.findPostByEntityId(context.imageEntity.postEntityId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, context.imageEntity.index
)
if (context.imageEntity.downloaded == context.imageEntity.size && context.imageEntity.size > 0) {
context.imageEntity.status = Status.FINISHED
post.done += 1
post.downloaded += context.imageEntity.size
dataAccessService.updatePost(post)
} else {
context.imageEntity.status = Status.ERROR
}
dataAccessService.updateImage(context.imageEntity)
}
} catch (e: Exception) {
if (context.stopped) {
return
}
context.imageEntity.status = Status.ERROR
dataAccessService.updateImage(context.imageEntity)
throw DownloadException(e)
}
}
private fun checkImageTypeAndRename(
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
) {
val existingExtension = getExtension(downloadedImage.name).lowercase()
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
val extension = when (downloadedImage.type) {
ImageMimeType.IMAGE_BMP -> "BMP"
ImageMimeType.IMAGE_GIF -> "GIF"
ImageMimeType.IMAGE_JPEG -> "JPG"
ImageMimeType.IMAGE_PNG -> "PNG"
ImageMimeType.IMAGE_WEBP -> "WEBP"
}
val filename = if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
sanitize(
fileNameWithoutExtension
)
}.$extension"
try {
val downloadDestinationFolder = Path.of(downloadDirectory)
Files.createDirectories(downloadDestinationFolder)
val finalFilename = "${
if (context.settings.downloadSettings.forceOrder) String.format(
"%03d_", index + 1
) else ""
}$filename"
context.imageEntity.filename = finalFilename
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
} catch (e: Exception) {
throw HostException("Failed to rename the image", e)
} finally {
try {
Files.delete(downloadedImage.path)
} catch (_: IOException) {
}
}
}
override fun run() {
try {
context.init()
if (context.stopped) {
return
}
download()
} finally {
context.completed = true
context.clear()
if (context.stopped && context.imageEntity.downloaded != context.imageEntity.size) {
context.imageEntity.status = Status.STOPPED
}
dataAccessService.updateImage(context.imageEntity)
}
}
fun stop() {
context.stopped = true
context.clear()
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
val that = other as ImageDownloadRunnable
return context.imageEntity.id == that.context.imageEntity.id
}
override fun hashCode(): Int {
return Objects.hash(context.imageEntity.id)
}
}
@@ -0,0 +1,3 @@
package me.vripper.services.download
internal data class ImageQueueElement(val imageEntityId: Long, val postEntityId: Long, val host: Byte)
@@ -0,0 +1,5 @@
package me.vripper.services.download
enum class MovePosition {
UP, DOWN, TOP, BOTTOM
}
@@ -0,0 +1,181 @@
package me.vripper.services.download
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import me.vripper.entities.Status
import me.vripper.event.ErrorCountEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.host.Host
import me.vripper.model.ErrorCount
import me.vripper.model.QueueState
import me.vripper.model.Rank
import me.vripper.services.DataAccessService
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.download.SharedLock.downloadManagerCondition
import me.vripper.services.download.SharedLock.downloadManagerLock
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.downloadRunner
import java.util.*
import kotlin.concurrent.withLock
internal class QueueManager(
private val dataAccessService: DataAccessService,
private val hosts: List<Host>,
private val settingsService: SettingsService,
private val retryPolicyService: RetryPolicyService,
private val eventBus: EventBus
) {
private val log by LoggerDelegate()
private val running: MutableList<ImageDownloadRunnable> = mutableListOf()
private val pending: MutableList<MutableList<ImageQueueElement>> = mutableListOf()
fun addPending(imageQueueElementList: List<ImageQueueElement>) {
pending.add(imageQueueElementList.toMutableList())
reportQueueState()
}
fun clearPending(postEntityId: Long? = null) {
val toProcess = if (postEntityId != null) pending.flatten()
.filter { it.postEntityId == postEntityId } else pending.flatten()
pending.forEach { it.removeAll(toProcess) }
pending.removeIf { it.isEmpty() }
reportQueueState()
}
fun runningTotalCount(): Map<Byte, Int> {
return hosts.associate { host ->
Pair(
host.hostId, running.count { it.context.imageEntity.host == host.hostId })
}
}
fun clearRunning(postEntityId: Long? = null) {
val toProcess =
if (postEntityId != null) running.filter { it.context.imageEntity.postEntityId == postEntityId } else running
toProcess.forEach { it.stop() }
while (toProcess.count { !it.context.completed } > 0) {
Thread.sleep(100)
}
running.removeAll(toProcess)
}
fun clearRunningRunnable(imageId: Long) {
running.removeIf { it.context.imageEntity.id == imageId }
}
fun accept(accepted: List<ImageQueueElement>) {
downloadManagerLock.withLock {
pending.forEach {
it.removeAll(accepted)
}
pending.removeIf { it.isEmpty() }
accepted.map {
ImageDownloadRunnable(
dataAccessService.findImageById(it.imageEntityId).orElseThrow(), settingsService.settings.copy()
)
}.forEach {
launch(it)
running.add(it)
}
reportQueueState()
}
}
fun pending(): List<ImageQueueElement> {
return pending.flatten()
}
fun isPending(postEntityId: Long): Boolean {
return pending.flatten().any { it.postEntityId == postEntityId }
}
fun isRunning(postEntityId: Long): Boolean {
return running.any { it.context.imageEntity.postEntityId == postEntityId }
}
fun move(postEntityId: Long, position: MovePosition) {
downloadManagerLock.withLock {
val index = pending.indexOfFirst { it.first().postEntityId == postEntityId }
when (position) {
MovePosition.UP -> {
val newIndex = if (index > 0) index - 1 else index
Collections.swap(pending, index, newIndex)
}
MovePosition.DOWN -> {
val newIndex = if (index < pending.size - 1) index + 1 else index
Collections.swap(pending, index, newIndex)
}
MovePosition.TOP -> {
if (index > 0) {
val element = pending.removeAt(index)
pending.addFirst(element)
}
}
MovePosition.BOTTOM -> {
if (index < pending.size - 1) {
val element = pending.removeAt(pending.size - 1)
pending.addLast(element)
}
}
}
reportQueueState()
}
}
fun getQueueState(): QueueState {
val ranks = downloadManagerLock.withLock {
pending.mapIndexed { index, elements -> Rank(elements.first().postEntityId, index + 1L) }
}
return QueueState(running.size, pending().size, ranks)
}
private fun launch(runnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${runnable.context.imageEntity.url}")
reportQueueState()
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${runnable.context.imageEntity.url}: "))
.with(downloadRunner).onFailure {
log.error(
"Failed to download ${runnable.context.imageEntity.url} after ${it.attemptCount} tries",
it.exception
)
val image = runnable.context.imageEntity
image.status = Status.ERROR
dataAccessService.updateImage(image)
}.onComplete {
afterJobFinish(runnable)
log.debug(
"Finished downloading ${runnable.context.imageEntity.url}"
)
}.runAsync(runnable)
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
downloadManagerLock.withLock {
val image = imageDownloadRunnable.context.imageEntity
clearRunningRunnable(imageDownloadRunnable.context.imageEntity.id)
if (!isPending(image.postEntityId) && !isRunning(image.postEntityId) && !imageDownloadRunnable.context.stopped) {
dataAccessService.finishPost(image.postEntityId, true)
}
reportQueueState()
log.debug("[{}] Event published: ErrorCountEvent after job finish", System.currentTimeMillis())
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataAccessService.countImagesInError())))
downloadManagerCondition.signal()
}
}
private fun reportQueueState() {
val queueState = getQueueState()
log.debug(
"[{}] Publishing event: QueueStateEvent(running={}, remaining={})",
System.currentTimeMillis(),
queueState.running,
queueState.remaining
)
eventBus.publishEvent(QueueStateEvent(queueState))
}
}
@@ -0,0 +1,9 @@
package me.vripper.services.download
import java.util.concurrent.locks.Condition
import java.util.concurrent.locks.ReentrantLock
object SharedLock {
val downloadManagerLock = ReentrantLock()
val downloadManagerCondition: Condition = downloadManagerLock.newCondition()
}
@@ -1,16 +1,20 @@
package me.vripper.tasks
import me.vripper.model.ThreadPostId
import me.vripper.services.*
import me.vripper.model.PostIdentifier
import me.vripper.services.DataAccessService
import me.vripper.services.MetadataService
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.services.download.DownloadService
import me.vripper.utilities.LoggerDelegate
import me.vripper.vgapi.PostItem
import me.vripper.vgapi.PostLookupAPIParser
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponent, Runnable {
internal class AddPostTask(private val items: List<PostIdentifier>) : KoinComponent, Runnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val dataAccessService: DataAccessService by inject()
private val settingsService: SettingsService by inject()
private val downloadService: DownloadService by inject()
private val threadCacheService: ThreadCacheService by inject()
@@ -20,19 +24,19 @@ internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponen
try {
Tasks.increment()
val toProcess = mutableListOf<PostItem>()
for ((threadId, postId) in items) {
if (dataTransaction.exists(postId)) {
for ((siteProxy, threadId, postId) in items) {
if (dataAccessService.existsPostId(postId)) {
log.info("Post $postId already loaded")
continue
}
val link =
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
"$siteProxy/threads/$threadId?p=$postId&viewfull=1#post$postId"
val cachedThread = threadCacheService.getIfPresent(threadId)
val threadItem = cachedThread ?:
PostLookupAPIParser(
threadId, postId
siteProxy, threadId, postId
).parse()
@@ -62,13 +66,13 @@ internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponen
}
val posts = try {
dataTransaction.newPosts(toProcess.toList())
dataAccessService.newPosts(toProcess.toList())
} catch (e: Exception) {
e.printStackTrace()
emptyList()
}
posts.forEach {
metadataService.fetchMetadata(it.postId)
metadataService.fetchMetadata(it)
}
if (settingsService.settings.downloadSettings.autoStart) {
downloadService.restartAll(posts)
@@ -1,9 +1,10 @@
package me.vripper.tasks
import me.vripper.entities.MetadataEntity
import me.vripper.entities.PostEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.services.DataTransaction
import me.vripper.services.DataAccessService
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
@@ -21,14 +22,14 @@ import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
internal class FetchMetadataTask(
private val postId: Long,
private val postEntity: PostEntity,
) : KoinComponent, Runnable {
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
private val log by LoggerDelegate()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
private val httpService: HTTPService by inject()
private val dataTransaction: DataTransaction by inject()
private val dataAccessService: DataAccessService by inject()
override fun run() {
try {
@@ -39,12 +40,12 @@ internal class FetchMetadataTask(
Tasks.increment()
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postId.toString()
"p", postEntity.vgPostId.toString()
)
}.build())
}.build()).also { it.setAbsoluteRequestUri(true) }
RequestLimit.getPermit(1)
log.debug("Requesting {}", httpGet.uri)
log.info("{}", httpGet)
val response = httpService.client.execute(httpGet, vgAuthService.createVgContext()) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
@@ -55,22 +56,22 @@ internal class FetchMetadataTask(
val document = HtmlUtils.clean(response)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
"//li[@id='post_${postEntity.vgPostId}']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'$postId'")
) ?: throw VripperException("Unable to find post #'${postEntity.vgPostId}'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'$postId'")
?: throw VripperException("Unable to find the poster for post #'${postEntity.vgPostId}'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
document, java.lang.String.format("//div[@id='post_message_%s']", postEntity.vgPostId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
dataTransaction.saveMetadata(metadataEntity)
val metadataEntity = MetadataEntity(postEntity.id, MetadataEntity.Data(postedBy, titles))
dataAccessService.saveMetadata(metadataEntity)
} finally {
Tasks.decrement()
}
@@ -36,13 +36,14 @@ internal class LeaveThanksTask(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.postId.toString()),
BasicNameValuePair("p", postEntity.vgPostId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
)
)
it.setAbsoluteRequestUri(true)
}
RequestLimit.getPermit(1)
log.info("Posting {}", postThanks.uri)
log.info("Posting {}", postThanks)
cm.client.execute(postThanks, context) { response ->
if (response.code / 100 != 2) {
throw VripperException("Unexpected response code '${response.code}' for $postThanks")
@@ -1,28 +1,30 @@
package me.vripper.tasks
import me.vripper.entities.ThreadEntity
import me.vripper.model.PostIdentifier
import me.vripper.model.Settings
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
import me.vripper.services.SettingsService
import me.vripper.services.DataAccessService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.taskRunner
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
internal class ThreadLookupTask(private val threadId: Long, private val settings: Settings) : KoinComponent, Runnable {
internal class ThreadLookupTask(
private val siteProxy: String,
private val threadId: Long,
private val settings: Settings
) : KoinComponent, Runnable {
private val log by LoggerDelegate()
private val dataTransaction by inject<DataTransaction>()
private val settingsService by inject<SettingsService>()
private val dataAccessService by inject<DataAccessService>()
private val threadCacheService by inject<ThreadCacheService>()
private val link: String = "${settingsService.settings.viperSettings.host}/threads/$threadId"
private val link: String = "$siteProxy/threads/$threadId"
override fun run() {
try {
Tasks.increment()
if (dataTransaction.findThreadByThreadId(threadId).isEmpty) {
val threadLookupResult = threadCacheService[threadId]
if (dataAccessService.findThreadByThreadId(threadId).isEmpty) {
val threadLookupResult = threadCacheService.loadThenCache(threadId, siteProxy)
if (threadLookupResult.error.isNotBlank()) {
log.error("Error loading $link: ${threadLookupResult.error}")
return
@@ -35,13 +37,13 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
taskRunner.submit(
AddPostTask(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
PostIdentifier(
siteProxy, it.threadId, it.postId
)
})
)
} else {
dataTransaction.save(
dataAccessService.save(
ThreadEntity(
title = threadLookupResult.title,
link = link,
@@ -36,3 +36,7 @@ fun String.hash256(): String {
val digest = md.digest(bytes)
return digest.fold("") { str, it -> str + "%02x".format(it) }
}
fun String.extractBaseUrl(): String {
return this.replaceFirst(Regex("(https?://[^/]+).*"), "$1")
}
@@ -18,7 +18,11 @@ import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import javax.xml.parsers.SAXParserFactory
internal class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
internal class PostLookupAPIParser(
private val siteProxy: String,
private val threadId: Long,
private val postId: Long
) : KoinComponent {
private val log by LoggerDelegate()
private val retryPolicyService: RetryPolicyService by inject()
private val httpService: HTTPService by inject()
@@ -32,8 +36,8 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
it.setParameter(
"p", postId.toString()
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
}.build()).also { it.setAbsoluteRequestUri(true) }
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler(siteProxy)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
@@ -42,7 +46,7 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
)
}.get(CheckedSupplier {
RequestLimit.getPermit(1)
log.info("Requesting {}", httpGet.uri)
log.info("{}", httpGet)
httpService.client.execute(
httpGet,
vgAuthService.createClickContext()
@@ -18,7 +18,7 @@ import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import javax.xml.parsers.SAXParserFactory
internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
internal class ThreadLookupAPIParser(private val siteProxy: String, private val threadId: Long) : KoinComponent {
private val log by LoggerDelegate()
private val cm: HTTPService by inject()
private val retryPolicyService: RetryPolicyService by inject()
@@ -33,8 +33,8 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
"t",
threadId.toString()
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
}.build()).also { it.setAbsoluteRequestUri(true) }
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler(siteProxy)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
@@ -44,7 +44,7 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
)
}.get(CheckedSupplier {
RequestLimit.getPermit(1)
log.info("Requesting {}", httpGet.uri)
log.info("{}", httpGet)
cm.client.execute(
httpGet, vgAuthService.createClickContext()
) { response ->
@@ -1,15 +1,12 @@
package me.vripper.vgapi
import me.vripper.host.Host
import me.vripper.services.SettingsService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.xml.sax.Attributes
import org.xml.sax.helpers.DefaultHandler
internal class ThreadLookupAPIResponseHandler : KoinComponent, DefaultHandler() {
internal class ThreadLookupAPIResponseHandler(private val siteProxy: String) : KoinComponent, DefaultHandler() {
private val supportedHosts: List<Host> = getKoin().getAll()
private val settingsService: SettingsService by inject()
private var error: String = ""
private val hostMap: MutableMap<Host, Int> = mutableMapOf()
private val postItemList: MutableList<PostItem> = mutableListOf()
@@ -74,7 +71,7 @@ internal class ThreadLookupAPIResponseHandler : KoinComponent, DefaultHandler()
postCounter,
postTitle,
imageItemList.size,
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId",
"$siteProxy/threads/$threadId?p=$postId&viewfull=1#post$postId",
hostMap.toMap().map { Pair(it.key.hostName, it.value) },
securityToken,
forum,
@@ -27,9 +27,14 @@ message DownloadSpeed {
message VGUser {
string user = 1;
}
message Rank {
int64 postEntityId = 1;
int64 rank = 2;
}
message QueueState {
int32 running = 1;
int32 remaining = 2;
repeated Rank rank = 3;
}
message ErrorCount {
int32 count = 1;
@@ -78,6 +83,16 @@ message Version {
message DBMigrationResponse {
string message = 1;
}
enum MovePositionEnum {
UP = 0;
DOWN = 1;
TOP = 2;
BOTTOM = 3;
}
message MovePositionMessage {
int64 postEntityId = 1;
MovePositionEnum position = 2;
}
service EndpointService {
rpc scanLinks (Links) returns (EmptyResponse) {}
@@ -94,6 +109,7 @@ service EndpointService {
rpc onDownloadSpeed (EmptyRequest) returns (stream DownloadSpeed) {}
rpc onVGUserUpdate (EmptyRequest) returns (stream VGUser) {}
rpc onQueueStateUpdate (EmptyRequest) returns (stream QueueState) {}
rpc getQueueState (EmptyRequest) returns (QueueState) {}
rpc onErrorCountUpdate (EmptyRequest) returns (stream ErrorCount) {}
rpc onTasksRunning (EmptyRequest) returns (stream TasksRunning) {}
rpc onUpdateMetadata (EmptyRequest) returns (stream Metadata) {}
@@ -120,4 +136,5 @@ service EndpointService {
rpc getVersion (EmptyRequest) returns (Version) {}
rpc dbMigration (EmptyRequest) returns (DBMigrationResponse) {}
rpc initLogger (EmptyRequest) returns (EmptyResponse) {}
rpc move (MovePositionMessage) returns (EmptyResponse) {}
}
+9 -10
View File
@@ -3,14 +3,13 @@ option java_package = "me.vripper.proto";
message Image {
int64 id = 1;
int64 postId = 2;
string url = 3;
string thumbUrl = 4;
int32 host = 5;
int32 index = 6;
int64 postIdRef = 7;
int64 size = 8;
int64 downloaded = 9;
string status = 10;
string filename = 11;
string url = 2;
string thumbUrl = 3;
int32 host = 4;
int32 index = 5;
int64 postIdRef = 6;
int64 size = 7;
int64 downloaded = 8;
string status = 9;
string filename = 10;
}
+5 -6
View File
@@ -17,10 +17,9 @@ message Post {
string folderName = 13;
string status = 14;
int32 done = 15;
int32 rank = 16;
int64 size = 17;
int64 downloaded = 18;
repeated string previews = 19;
string postedBy = 20;
repeated string resolvedNames = 21;
int64 size = 16;
int64 downloaded = 17;
repeated string previews = 18;
string postedBy = 19;
repeated string resolvedNames = 20;
}
@@ -36,9 +36,14 @@ message SystemSettings {
int32 maxEventLog = 4;
}
message HostSettingsMap {
map<string, string> settings = 1;
}
message Settings {
ConnectionSettings connectionSettings = 1;
DownloadSettings downloadSettings = 2;
ViperSettings viperSettings = 3;
SystemSettings systemSettings = 4;
map<string, HostSettingsMap> hostSettings = 5;
}
@@ -142,4 +142,15 @@
</column>
</createTable>
</changeSet>
<changeSet author="vripper" id="0007">
<renameColumn oldColumnName="POST_ID" newColumnName="POST_ID_REF" tableName="METADATA"/>
</changeSet>
<changeSet author="vripper" id="0009">
<dropIndex tableName="IMAGE" indexName="IMAGE_POST_ID_IDX"/>
</changeSet>
<changeSet author="vripper" id="0010">
<sql>
ALTER TABLE IMAGE DROP COLUMN POST_ID
</sql>
</changeSet>
</databaseChangeLog>
+15 -3
View File
@@ -25,6 +25,18 @@
<appender name="RING" class="me.vripper.RingAppender"/>
<appender name="ASYNC-STDOUT" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="STDOUT"/>
</appender>
<appender name="ASYNC-FILE" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="FILE"/>
</appender>
<appender name="ASYNC-RING" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="RING"/>
</appender>
<logger name="me.vripper" level="info"/>
<logger name="Exposed" level="off"/>
<logger name="org.springframework.web.socket.config.WebSocketMessageBrokerStats" level="off"/>
@@ -32,8 +44,8 @@
<Logger name="org.apache.hc.client5.http.wire" level="off"/>
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="FILE"/>
<appender-ref ref="RING"/>
<appender-ref ref="ASYNC-STDOUT"/>
<appender-ref ref="ASYNC-FILE"/>
<appender-ref ref="ASYNC-RING"/>
</root>
</configuration>
+1 -1
View File
@@ -1 +1 @@
${project.version}
${app-version}
+7 -2
View File
@@ -16,7 +16,7 @@
<description>vripper-gui</description>
<properties>
<javafx.version>23.0.1</javafx.version>
<javafx.version>25</javafx.version>
</properties>
<dependencies>
@@ -114,7 +114,7 @@
<dependency>
<groupId>io.github.mkpaz</groupId>
<artifactId>atlantafx-base</artifactId>
<version>2.0.1</version>
<version>2.1.0</version>
</dependency>
<dependency>
<groupId>org.kordamp.ikonli</groupId>
@@ -147,6 +147,11 @@
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
</dependency>
<dependency>
<groupId>com.google.jimfs</groupId>
<artifactId>jimfs</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
<build>
@@ -1,7 +1,6 @@
package me.vripper.gui
import atlantafx.base.theme.CupertinoDark
import atlantafx.base.theme.CupertinoLight
import atlantafx.base.theme.*
import javafx.application.Application
import javafx.scene.image.Image
import javafx.stage.Stage
@@ -38,13 +37,19 @@ class VripperGuiApplication : App(
}
override fun start(stage: Stage) {
Thread.setDefaultUncaughtExceptionHandler(Thread.UncaughtExceptionHandler { t, e ->
Thread.setDefaultUncaughtExceptionHandler { t, e ->
log.error("Thread $t threw an exception: ${e.message}", e)
})
if (widgetsController.currentSettings.darkMode) {
setUserAgentStylesheet(CupertinoDark().userAgentStylesheet)
} else {
setUserAgentStylesheet(CupertinoLight().userAgentStylesheet)
}
when (widgetsController.currentSettings.theme) {
"CupertinoLight" -> setUserAgentStylesheet(CupertinoLight().userAgentStylesheet)
"CupertinoDark" -> setUserAgentStylesheet(CupertinoDark().userAgentStylesheet)
"NordLight" -> setUserAgentStylesheet(NordLight().userAgentStylesheet)
"NordDark" -> setUserAgentStylesheet(NordDark().userAgentStylesheet)
"PrimerLight" -> setUserAgentStylesheet(PrimerLight().userAgentStylesheet)
"PrimerDark" -> setUserAgentStylesheet(PrimerDark().userAgentStylesheet)
"Dracula" -> setUserAgentStylesheet(Dracula().userAgentStylesheet)
else -> setUserAgentStylesheet(CupertinoLight().userAgentStylesheet)
}
with(stage) {
width = widgetsController.currentSettings.width
@@ -1,17 +1,16 @@
package me.vripper.gui.components.cells
import javafx.collections.ObservableList
import javafx.geometry.Pos
import javafx.scene.control.TableCell
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
class PreviewTableCell<T> : TableCell<T, ObservableList<String>>() {
class PreviewTableCell<T, V> : TableCell<T, V>() {
init {
alignment = Pos.CENTER
}
override fun updateItem(item: ObservableList<String>?, empty: Boolean) {
override fun updateItem(item: V, empty: Boolean) {
super.updateItem(item, empty)
graphic = if (!empty) {
FontIcon.of(Feather.IMAGE)
@@ -88,19 +88,6 @@ class AboutFragment : Fragment("About") {
isEditable = false
}
}
field("Previews Path") {
textfield(widgetsController.currentSettings.cachePathProperty) {
isEditable = false
}
button("Browse") {
action {
val directory = chooseDirectory(title = "Select previews folder")
if (directory != null) {
widgetsController.currentSettings.cachePathProperty.set(directory.path)
}
}
}
}
}
}
}
@@ -31,8 +31,10 @@ class AddLinksFragment : Fragment("Add thread links") {
action {
coroutineScope.launch {
postController.scan(textAreaProperty.value)
runLater {
close()
}
}
close()
}
}
}
@@ -1,63 +1,51 @@
package me.vripper.gui.components.fragments
import javafx.scene.control.Spinner
import kotlinx.coroutines.*
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.model.settings.ConnectionSettingsModel
import me.vripper.model.ConnectionSettings
import tornadofx.*
class ConnectionSettingsFragment : Fragment("Connection Settings") {
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var connectionSettings: ConnectionSettings
val connectionSettings: ConnectionSettings by param()
val connectionSettingsModel = ConnectionSettingsModel()
override val root = vbox {}
init {
coroutineScope.launch {
connectionSettings = settingsController.findConnectionSettings()
runLater {
with(root) {
form {
fieldset {
field("Concurrent downloads per host") {
add(Spinner<Int>(1, 4, connectionSettings.maxConcurrentPerHost).apply {
connectionSettingsModel.maxThreadsProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Global concurrent downloads") {
add(Spinner<Int>(0, 24, connectionSettings.maxGlobalConcurrent).apply {
connectionSettingsModel.maxTotalThreadsProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Connection timeout (s)") {
add(Spinner<Int>(1, 300, connectionSettings.timeout).apply {
connectionSettingsModel.timeoutProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Maximum attempts") {
add(Spinner<Int>(1, 10, connectionSettings.maxAttempts).apply {
connectionSettingsModel.maxAttemptsProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
}
with(root) {
form {
fieldset {
field("Concurrent downloads per host") {
add(Spinner<Int>(1, 4, connectionSettings.maxConcurrentPerHost).apply {
connectionSettingsModel.maxThreadsProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Global concurrent downloads") {
add(Spinner<Int>(0, 24, connectionSettings.maxGlobalConcurrent).apply {
connectionSettingsModel.maxTotalThreadsProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Connection timeout (s)") {
add(Spinner<Int>(1, 300, connectionSettings.timeout).apply {
connectionSettingsModel.timeoutProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Maximum attempts") {
add(Spinner<Int>(1, 10, connectionSettings.maxAttempts).apply {
connectionSettingsModel.maxAttemptsProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
}
}
}
}
override fun onUndock() {
coroutineScope.cancel()
}
}
@@ -2,8 +2,6 @@ package me.vripper.gui.components.fragments
import atlantafx.base.util.IntegerStringConverter
import javafx.scene.control.Spinner
import kotlinx.coroutines.*
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.controller.WidgetsController
import me.vripper.gui.model.settings.DownloadSettingsModel
import me.vripper.model.DownloadSettings
@@ -11,89 +9,76 @@ import tornadofx.*
class DownloadSettingsFragment : Fragment("Download Settings") {
private val settingsController: SettingsController by inject()
val downloadSettings: DownloadSettings by param()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var downloadSettings: DownloadSettings
val downloadSettingsModel = DownloadSettingsModel()
override val root = vbox {}
init {
coroutineScope.launch {
downloadSettings = settingsController.findDownloadSettings() ?: DownloadSettings()
downloadSettingsModel.downloadPath = downloadSettings.downloadPath
downloadSettingsModel.autoStart = downloadSettings.autoStart
downloadSettingsModel.forceOrder = downloadSettings.forceOrder
downloadSettingsModel.forumSubfolder = downloadSettings.forumSubDirectory
downloadSettingsModel.threadSubLocation = downloadSettings.threadSubLocation
downloadSettingsModel.clearCompleted = downloadSettings.clearCompleted
downloadSettingsModel.appendPostId = downloadSettings.appendPostId
runLater {
with(root) {
form {
fieldset {
field("Download Path") {
textfield(downloadSettingsModel.downloadPathProperty) {
editableWhen(widgetsController.currentSettings.localSessionProperty.not())
}
button("Browse") {
visibleWhen(widgetsController.currentSettings.localSessionProperty)
action {
val directory = chooseDirectory(title = "Select download folder")
if (directory != null) {
downloadSettingsModel.downloadPathProperty.set(directory.path)
}
}
}
}
field("Auto start downloads") {
checkbox {
bind(downloadSettingsModel.autoStartProperty)
}
}
field("Auto queue thread if post count is below or equal to") {
add(Spinner<Int>(1, Int.MAX_VALUE, downloadSettings.autoQueueThreshold).apply {
downloadSettingsModel.autoQueueThresholdProperty.bind(valueProperty())
isEditable = true
IntegerStringConverter.createFor(this)
})
}
field("Organize by category") {
checkbox {
bind(downloadSettingsModel.forumSubfolderProperty)
}
}
field("Organize by thread") {
checkbox {
bind(downloadSettingsModel.threadSubLocationProperty)
}
}
field("Order images") {
checkbox {
bind(downloadSettingsModel.forceOrderProperty)
}
}
field("Append post id to download folder") {
checkbox {
bind(downloadSettingsModel.appendPostIdProperty)
}
}
field("Clear Finished") {
checkbox {
bind(downloadSettingsModel.clearCompletedProperty)
downloadSettingsModel.downloadPath = downloadSettings.downloadPath
downloadSettingsModel.autoStart = downloadSettings.autoStart
downloadSettingsModel.forceOrder = downloadSettings.forceOrder
downloadSettingsModel.forumSubfolder = downloadSettings.forumSubDirectory
downloadSettingsModel.threadSubLocation = downloadSettings.threadSubLocation
downloadSettingsModel.clearCompleted = downloadSettings.clearCompleted
downloadSettingsModel.appendPostId = downloadSettings.appendPostId
with(root) {
form {
fieldset {
field("Download Path") {
textfield(downloadSettingsModel.downloadPathProperty) {
editableWhen(widgetsController.currentSettings.localSessionProperty.not())
}
button("Browse") {
visibleWhen(widgetsController.currentSettings.localSessionProperty)
action {
val directory = chooseDirectory(title = "Select download folder")
if (directory != null) {
downloadSettingsModel.downloadPathProperty.set(directory.path)
}
}
}
}
field("Auto start downloads") {
checkbox {
bind(downloadSettingsModel.autoStartProperty)
}
}
field("Auto queue thread if post count is below or equal to") {
add(Spinner<Int>(1, Int.MAX_VALUE, downloadSettings.autoQueueThreshold).apply {
downloadSettingsModel.autoQueueThresholdProperty.bind(valueProperty())
isEditable = true
IntegerStringConverter.createFor(this)
})
}
field("Organize by category") {
checkbox {
bind(downloadSettingsModel.forumSubfolderProperty)
}
}
field("Organize by thread") {
checkbox {
bind(downloadSettingsModel.threadSubLocationProperty)
}
}
field("Order images") {
checkbox {
bind(downloadSettingsModel.forceOrderProperty)
}
}
field("Append post id to download folder") {
checkbox {
bind(downloadSettingsModel.appendPostIdProperty)
}
}
field("Clear Finished") {
checkbox {
bind(downloadSettingsModel.clearCompletedProperty)
}
}
}
}
}
}
override fun onUndock() {
coroutineScope.cancel()
}
}
@@ -0,0 +1,76 @@
package me.vripper.gui.components.fragments
import atlantafx.base.util.IntegerStringConverter
import javafx.scene.control.Spinner
import me.vripper.gui.model.settings.HostSettingsModel
import me.vripper.model.HostName
import me.vripper.model.HostSettingKey
import me.vripper.model.SettingType
import tornadofx.*
class HostSettingsFragment : Fragment("Host Settings") {
val hostSettings: Map<HostName, Map<HostSettingKey, String>> by param()
val hostSettingsModels: List<HostSettingsModel>
override val root = vbox {}
init {
val mutableHostSettingsModels = mutableListOf<HostSettingsModel>()
// Build models dynamically from hostSettings map
hostSettings.forEach { (hostName, settingsMap) ->
settingsMap.forEach { (settingKey, settingValue) ->
try {
val key = settingKey
val model = HostSettingsModel(
host = hostName,
settingKey = key,
initialValue = settingValue
)
mutableHostSettingsModels.add(model)
} catch (e: IllegalArgumentException) {
// Skip unknown settings
}
}
}
hostSettingsModels = mutableHostSettingsModels.toList()
// Build UI dynamically
with(root) {
val groupedByHost = hostSettingsModels.groupBy { it.host }
form {
groupedByHost.forEach { (hostName, settings) ->
fieldset(hostName.name) {
settings.forEach { model ->
field(
model.settingKey.name.replace("_", " ").lowercase()
.replaceFirstChar { it.uppercase() }) {
when (model.settingKey.type) {
SettingType.STRING -> textfield(model.valueProperty as javafx.beans.property.StringProperty)
SettingType.BOOLEAN -> checkbox(
"",
model.valueProperty as javafx.beans.property.BooleanProperty
)
SettingType.INT -> {
add(Spinner<Int>(Int.MIN_VALUE, Int.MAX_VALUE, model.getValue().toInt()).apply {
valueProperty().onChange {
(model.valueProperty as javafx.beans.property.IntegerProperty).value =
it ?: 0
}
isEditable = true
IntegerStringConverter.createFor(this)
})
}
}
}
}
}
}
}
}
}
}
@@ -12,7 +12,7 @@ import tornadofx.*
class RenameFragment : Fragment("Rename download post") {
val postId: Long by param()
val id: Long by param()
val name: String by param()
val altTitles: List<String> by param()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
@@ -43,7 +43,7 @@ class RenameFragment : Fragment("Rename download post") {
disableWhen(comboBox.editor.textProperty().isEmpty)
action {
coroutineScope.launch {
postController.rename(postId, comboBox.editor.text.trim())
postController.rename(this@RenameFragment.id, comboBox.editor.text.trim())
}
close()
}
@@ -9,17 +9,15 @@ import javafx.scene.control.ToggleGroup
import javafx.scene.layout.VBox
import kotlinx.coroutines.runBlocking
import me.vripper.gui.VripperGuiApplication
import me.vripper.gui.components.views.AppView
import me.vripper.gui.components.views.LoadingView
import me.vripper.gui.controller.WidgetsController
import me.vripper.gui.event.GuiEventBus
import me.vripper.gui.services.GrpcEndpointService
import me.vripper.gui.utils.AppEndpointManager
import me.vripper.listeners.AppManager
import tornadofx.*
class SessionFragment : Fragment("Change Session") {
private val widgetsController: WidgetsController by inject()
private val grpcEndpointService: GrpcEndpointService by di("remoteAppEndpointService")
private val toggleGroup = ToggleGroup()
override val root = VBox().apply {
alignment = Pos.CENTER
@@ -32,11 +30,11 @@ class SessionFragment : Fragment("Change Session") {
form {
fieldset("Source") {
radiobutton(text = "Start Local Session", group = toggleGroup) {
id = "localSession"
this.id = "localSession"
isSelected = widgetsController.currentSettings.localSession
}
val remoteRadio = radiobutton(text = "Connect to Remote Session", group = toggleGroup) {
id = "remoteSession"
this.id = "remoteSession"
isSelected = !widgetsController.currentSettings.localSession
}
field("Host") {
@@ -65,37 +63,28 @@ class SessionFragment : Fragment("Change Session") {
addClass(Styles.ACCENT)
isDefaultButton = true
action {
val selectedToggle = toggleGroup.selectedToggle
runBlocking {
GuiEventBus.publishEvent(GuiEventBus.ChangingSession)
AppManager.stop()
grpcEndpointService.disconnect()
}
when ((selectedToggle as RadioButton).id) {
"localSession" -> {
runBlocking {
widgetsController.currentSettings.localSession = true
AppEndpointManager.set(GuiEventBus.LocalSession)
AppManager.start()
GuiEventBus.publishEvent(GuiEventBus.LocalSession)
}
}
"remoteSession" -> {
runBlocking {
widgetsController.currentSettings.localSession = false
AppEndpointManager.set(GuiEventBus.RemoteSession)
grpcEndpointService.connect(
widgetsController.currentSettings.remoteSessionModel.host,
widgetsController.currentSettings.remoteSessionModel.port,
widgetsController.currentSettings.remoteSessionModel.passcode,
)
GuiEventBus.publishEvent(GuiEventBus.RemoteSession)
}
}
else -> VripperGuiApplication.APP_INSTANCE.stop()
val selectedToggle = toggleGroup.selectedToggle
runLater {
find<AppView>().replaceWith(find<LoadingView>())
}
runLater {
when ((selectedToggle as RadioButton).id) {
"localSession" -> {
widgetsController.currentSettings.localSession = true
}
"remoteSession" -> {
widgetsController.currentSettings.localSession = false
}
else -> VripperGuiApplication.APP_INSTANCE.stop()
}
runBlocking {
GuiEventBus.publishEvent(GuiEventBus.ApplicationInitialized(emptyList()))
}
close()
}
}
@@ -9,6 +9,10 @@ import javafx.scene.layout.VBox
import kotlinx.coroutines.*
import me.vripper.exception.ValidationException
import me.vripper.gui.controller.SettingsController
import me.vripper.model.ConnectionSettings
import me.vripper.model.DownloadSettings
import me.vripper.model.SystemSettings
import me.vripper.model.ViperSettings
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
@@ -16,12 +20,24 @@ import tornadofx.*
class SettingsFragment : Fragment("Settings") {
val downloadSettings: DownloadSettings by param()
val connectionSettings: ConnectionSettings by param()
val viperSettings: ViperSettings by param()
val systemSettings: SystemSettings by param()
val hostSettings: Map<String, Map<String, String>> by param()
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val downloadSettingsFragment: DownloadSettingsFragment = find()
private val connectionSettingsFragment: ConnectionSettingsFragment = find()
private val viperSettingsFragment: ViperSettingsFragment = find()
private val systemSettingsFragment: SystemSettingsFragment = find()
private val downloadSettingsFragment: DownloadSettingsFragment =
find(mapOf(DownloadSettingsFragment::downloadSettings to downloadSettings))
private val connectionSettingsFragment: ConnectionSettingsFragment =
find(mapOf(ConnectionSettingsFragment::connectionSettings to connectionSettings))
private val viperSettingsFragment: ViperSettingsFragment =
find(mapOf(ViperSettingsFragment::viperSettings to viperSettings))
private val systemSettingsFragment: SystemSettingsFragment =
find(mapOf(SystemSettingsFragment::systemSettings to systemSettings))
private val hostSettingsFragment: HostSettingsFragment =
find(mapOf(HostSettingsFragment::hostSettings to hostSettings))
override val root = vbox(alignment = Pos.CENTER_RIGHT) {
spacing = 5.0
@@ -31,6 +47,7 @@ class SettingsFragment : Fragment("Settings") {
minWidth = 100.0
minHeight = 100.0
prefHeight = 400.0
prefWidth = 800.0
tab(downloadSettingsFragment.title) {
add(downloadSettingsFragment)
graphic = FontIcon.of(Feather.FOLDER)
@@ -47,6 +64,10 @@ class SettingsFragment : Fragment("Settings") {
add(viperSettingsFragment)
graphic = FontIcon.of(Feather.LINK_2)
}
tab(hostSettingsFragment.title) {
add(hostSettingsFragment)
graphic = FontIcon.of(Feather.IMAGE)
}
}
borderpane {
right {
@@ -62,7 +83,8 @@ class SettingsFragment : Fragment("Settings") {
downloadSettingsFragment.downloadSettingsModel,
connectionSettingsFragment.connectionSettingsModel,
viperSettingsFragment.viperSettingsModel,
systemSettingsFragment.systemSettingsModel
systemSettingsFragment.systemSettingsModel,
hostSettingsFragment.hostSettingsModels
)
runLater {
close()
@@ -3,73 +3,74 @@ package me.vripper.gui.components.fragments
import atlantafx.base.controls.ToggleSwitch
import atlantafx.base.util.IntegerStringConverter
import javafx.scene.control.Spinner
import kotlinx.coroutines.*
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.controller.WidgetsController
import me.vripper.gui.model.settings.SystemSettingsModel
import me.vripper.model.SystemSettings
import tornadofx.*
class SystemSettingsFragment : Fragment("System Settings") {
private val settingsController: SettingsController by inject()
val systemSettings: SystemSettings by param()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var systemSettings: SystemSettings
val systemSettingsModel = SystemSettingsModel()
override val root = vbox {}
init {
coroutineScope.launch {
systemSettings = settingsController.findSystemSettings()
systemSettingsModel.tempPath = systemSettings.tempPath
runLater {
with(root) {
form {
systemSettingsModel.tempPath = systemSettings.tempPath
systemSettingsModel.logEntries = systemSettings.maxEventLog
systemSettingsModel.enable = systemSettings.enableClipboardMonitoring
systemSettingsModel.pollingRate = systemSettings.clipboardPollingRate
with(root) {
form {
fieldset {
field("Temporary Path") {
textfield(systemSettingsModel.tempPathProperty) {
editableWhen(widgetsController.currentSettings.localSessionProperty.not())
}
button("Browse") {
visibleWhen(widgetsController.currentSettings.localSessionProperty)
action {
val directory = chooseDirectory(title = "Select temporary folder")
if (directory != null) {
systemSettingsModel.tempPathProperty.set(directory.path)
}
}
}
}
field("Max log entries") {
add(Spinner<Int>(10, 10000, systemSettingsModel.logEntries).apply {
valueProperty().onChange {
systemSettingsModel.logEntries = it!!
}
isEditable = true
IntegerStringConverter.createFor(this)
})
}
fieldset {
field("Clipboard monitoring") {
add(ToggleSwitch().apply {
isSelected = systemSettingsModel.enable
selectedProperty().onChange {
systemSettingsModel.enable = it
}
})
}
fieldset {
field("Temporary Path") {
textfield(systemSettingsModel.tempPathProperty) {
editableWhen(widgetsController.currentSettings.localSessionProperty.not())
}
button("Browse") {
visibleWhen(widgetsController.currentSettings.localSessionProperty)
action {
val directory = chooseDirectory(title = "Select temporary folder")
if (directory != null) {
systemSettingsModel.tempPathProperty.set(directory.path)
visibleWhen(systemSettingsModel.enableProperty)
field("Polling rate (ms)") {
add(
Spinner<Int>(
500,
Int.MAX_VALUE,
systemSettingsModel.pollingRate
).apply {
valueProperty().onChange {
systemSettingsModel.pollingRate = it!!
}
}
}
}
field("Max log entries") {
add(Spinner<Int>(10, 10000, systemSettings.maxEventLog).apply {
systemSettingsModel.logEntriesProperty.bind(valueProperty())
isEditable = true
IntegerStringConverter.createFor(this)
})
}
fieldset {
field("Clipboard monitoring") {
add(ToggleSwitch().apply {
isSelected = systemSettings.enableClipboardMonitoring
systemSettingsModel.enableProperty.bind(selectedProperty())
isEditable = true
IntegerStringConverter.createFor(this)
})
}
fieldset {
visibleWhen(systemSettingsModel.enableProperty)
field("Polling rate (ms)") {
add(
Spinner<Int>(
500,
kotlin.Int.MAX_VALUE,
systemSettings.clipboardPollingRate
).apply {
systemSettingsModel.pollingRateProperty.bind(valueProperty())
isEditable = true
IntegerStringConverter.createFor(this)
})
}
}
}
}
}
@@ -77,8 +78,4 @@ class SystemSettingsFragment : Fragment("System Settings") {
}
}
}
override fun onUndock() {
coroutineScope.cancel()
}
}
@@ -2,6 +2,7 @@ package me.vripper.gui.components.fragments
import atlantafx.base.theme.Styles
import javafx.beans.property.SimpleStringProperty
import javafx.collections.ObservableList
import javafx.event.EventHandler
import javafx.geometry.Insets
import javafx.geometry.Pos
@@ -22,15 +23,15 @@ import me.vripper.gui.utils.openLink
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
import kotlin.io.path.Path
class ThreadSelectionTableFragment : Fragment("Thread") {
private lateinit var tableView: TableView<ThreadSelectionModel>
override val root = vbox(alignment = Pos.CENTER_RIGHT, spacing = 15.0)
private val tableView: TableView<ThreadSelectionModel>
private val threadController: ThreadController by inject()
private val widgetsController: WidgetsController by inject()
private var items = SortedFilteredList<ThreadSelectionModel>()
private var preview: Preview? = null
private val preview: Preview = Preview(currentStage!!)
private val searchInput = SimpleStringProperty()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val threadId: Long by param()
@@ -48,182 +49,185 @@ class ThreadSelectionTableFragment : Fragment("Thread") {
}
override fun onUndock() {
preview.destroy()
coroutineScope.cancel()
}
override val root = vbox(alignment = Pos.CENTER_RIGHT, spacing = 15.0) {
hbox(spacing = 5, alignment = Pos.BASELINE_LEFT) {
padding = Insets(10.0, 5.0, 0.0, 5.0)
textfield(searchInput) {
promptText = "Search"
hgrow = Priority.ALWAYS
init {
with(root) {
hbox(spacing = 5, alignment = Pos.BASELINE_LEFT) {
padding = Insets(10.0, 5.0, 0.0, 5.0)
textfield(searchInput) {
promptText = "Search"
hgrow = Priority.ALWAYS
}
}
}
tableView = tableview(items) {
isTableMenuButtonVisible = true
addClass(Styles.DENSE)
selectionModel.selectionMode = SelectionMode.MULTIPLE
setRowFactory {
val tableRow = TableRow<ThreadSelectionModel>()
tableView = tableview(items) {
isTableMenuButtonVisible = true
addClass(Styles.DENSE)
selectionModel.selectionMode = SelectionMode.MULTIPLE
setRowFactory {
val tableRow = TableRow<ThreadSelectionModel>()
tableRow.setOnMouseClicked {
if (it.button.equals(MouseButton.PRIMARY) && it.clickCount == 2 && tableRow.item != null) {
coroutineScope.launch {
async { threadController.download(listOf(tableRow.item)) }.await()
runLater {
close()
tableRow.setOnMouseClicked {
if (it.button.equals(MouseButton.PRIMARY) && it.clickCount == 2 && tableRow.item != null) {
coroutineScope.launch {
async { threadController.download(listOf(tableRow.item)) }.await()
runLater {
close()
}
}
}
}
}
val urlItem = MenuItem("Open link").apply {
setOnAction {
openLink(tableRow.item.url)
}
graphic = FontIcon.of(Feather.LINK)
}
val contextMenu = ContextMenu()
contextMenu.items.addAll(urlItem)
tableRow.contextMenuProperty().bind(
tableRow.emptyProperty()
.map { empty -> if (empty) null else contextMenu })
tableRow
}
column("Preview", ThreadSelectionModel::previewListProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.previewProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.previewProperty.set(
it
)
}
prefWidth = 100.0
cellFactory = Callback {
val cell = PreviewTableCell<ThreadSelectionModel>()
cell.onMouseExited = EventHandler {
preview?.hide()
}
cell.onMouseMoved = EventHandler {
preview?.previewPopup?.apply {
x = it.screenX + 20
y = it.screenY + 10
val urlItem = MenuItem("Open link").apply {
setOnAction {
openLink(tableRow.item.url)
}
graphic = FontIcon.of(Feather.LINK)
}
cell.onMouseEntered = EventHandler { mouseEvent ->
preview?.hide()
if (cell.tableRow.item != null && cell.tableRow.item.previewList.isNotEmpty()) {
preview = Preview(
currentStage!!,
cell.tableRow.item.previewList,
Path(widgetsController.currentSettings.cachePath)
)
preview?.previewPopup?.apply {
x = mouseEvent.screenX + 20
y = mouseEvent.screenY + 10
val contextMenu = ContextMenu()
contextMenu.items.addAll(urlItem)
tableRow.contextMenuProperty().bind(
tableRow.emptyProperty()
.map { empty -> if (empty) null else contextMenu })
tableRow
}
column("Preview", ThreadSelectionModel::previewListProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.previewProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.previewProperty.set(
it
)
}
prefWidth = 100.0
cellFactory = Callback {
val cell = PreviewTableCell<ThreadSelectionModel, ObservableList<String>>()
cell.onMouseExited = EventHandler {
preview.cleanup()
}
cell.onMouseMoved = EventHandler {
preview.previewPopup.apply {
x = it.screenX + 20
y = it.screenY + 10
}
}
}
cell
}
}
column("Post Index", ThreadSelectionModel::indexProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.indexProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.indexProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.index
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.index = it as Double
cell.onMouseEntered = EventHandler { mouseEvent ->
preview.cleanup()
if (cell.tableRow.item != null && cell.tableRow.item.previewList.isNotEmpty()) {
preview.display(
cell.tableRow.item.threadId,
cell.tableRow.item.previewList
)
preview.previewPopup.apply {
x = mouseEvent.screenX + 20
y = mouseEvent.screenY + 10
}
}
}
cell
}
}
sortOrder.add(this)
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, Number?>().apply { alignment = Pos.CENTER_LEFT }
}
}
column("Title", ThreadSelectionModel::titleProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.titleProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.titleProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.title
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.title = it as Double
column("Post Index", ThreadSelectionModel::indexProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.indexProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.indexProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.index
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.index = it as Double
}
}
sortOrder.add(this)
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, Number?>().apply { alignment = Pos.CENTER_LEFT }
}
}
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, String?>().apply { alignment = Pos.CENTER_LEFT }
}
}
column("URL", ThreadSelectionModel::urlProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.linkProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.linkProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.link
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.link = it as Double
column("Title", ThreadSelectionModel::titleProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.titleProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.titleProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.title
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.title = it as Double
}
}
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, String?>().apply { alignment = Pos.CENTER_LEFT }
}
}
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, String?>().apply { alignment = Pos.CENTER_LEFT }
}
}
column("Hosts", ThreadSelectionModel::hostsProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.hostsProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.hostsProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.hosts
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.hosts = it as Double
column("URL", ThreadSelectionModel::urlProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.linkProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.linkProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.link
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.link = it as Double
}
}
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, String?>().apply { alignment = Pos.CENTER_LEFT }
}
}
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, String?>().apply { alignment = Pos.CENTER_LEFT }
column("Hosts", ThreadSelectionModel::hostsProperty) {
isVisible = widgetsController.currentSettings.threadSelectionColumnsModel.hostsProperty.get()
visibleProperty().onChange {
widgetsController.currentSettings.threadSelectionColumnsModel.hostsProperty.set(
it
)
}
prefWidth = widgetsController.currentSettings.threadSelectionColumnsWidthModel.hosts
coroutineScope.launch {
widthProperty().asFlow().debounce(200).collect {
widgetsController.currentSettings.threadSelectionColumnsWidthModel.hosts = it as Double
}
}
cellFactory = Callback {
TextFieldTableCell<ThreadSelectionModel?, String?>().apply { alignment = Pos.CENTER_LEFT }
}
}
}
}
borderpane {
right {
padding = insets(top = 0, right = 5, bottom = 5, left = 5)
button("Download") {
graphic = FontIcon.of(Feather.DOWNLOAD)
addClass(Styles.ACCENT)
isDefaultButton = true
tooltip("Download selected posts")
enableWhen { tableView.selectionModel.selectedItems.sizeProperty.greaterThan(0) }
action {
coroutineScope.launch {
async { threadController.download(tableView.selectionModel.selectedItems) }.await()
runLater {
close()
borderpane {
right {
padding = insets(top = 0, right = 5, bottom = 5, left = 5)
button("Download") {
graphic = FontIcon.of(Feather.DOWNLOAD)
addClass(Styles.ACCENT)
isDefaultButton = true
tooltip("Download selected posts")
enableWhen { tableView.selectionModel.selectedItems.sizeProperty.greaterThan(0) }
action {
coroutineScope.launch {
async { threadController.download(tableView.selectionModel.selectedItems) }.await()
runLater {
close()
}
}
}
}
}
}
}
searchInput.onChange { search ->
if (search != null) {
items.predicate = { it.title.contains(search, true) }
searchInput.onChange { search ->
if (search != null) {
items.predicate = { it.title.contains(search, true) }
}
}
items.bindTo(tableView)
}
items.bindTo(tableView)
}
}
@@ -3,69 +3,66 @@ package me.vripper.gui.components.fragments
import atlantafx.base.controls.ToggleSwitch
import javafx.collections.FXCollections
import javafx.scene.control.Spinner
import kotlinx.coroutines.*
import kotlinx.coroutines.runBlocking
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.model.settings.ViperSettingsModel
import me.vripper.model.ViperSettings
import tornadofx.*
class ViperSettingsFragment : Fragment("Viper Settings") {
val viperSettings: ViperSettings by param()
private val settingsController: SettingsController by inject()
private val proxies = FXCollections.observableArrayList<String>()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var viperGirlsSettings: ViperSettings
val viperSettingsModel = ViperSettingsModel()
override val root = vbox {}
init {
coroutineScope.launch {
viperGirlsSettings = settingsController.findViperGirlsSettings()
viperSettingsModel.username = viperGirlsSettings.username
viperSettingsModel.password = viperGirlsSettings.password
viperSettingsModel.thanks = viperGirlsSettings.thanks
viperSettingsModel.host = viperGirlsSettings.host
viperSettingsModel.requestLimit = viperGirlsSettings.requestLimit
viperSettingsModel.fetchMetadata = viperGirlsSettings.fetchMetadata
proxies.addAll(settingsController.getProxies())
runLater {
with(root) {
form {
fieldset {
field("Viper domain") {
combobox(viperSettingsModel.hostProperty, proxies)
viperSettingsModel.username = viperSettings.username
viperSettingsModel.password = viperSettings.password
viperSettingsModel.thanks = viperSettings.thanks
viperSettingsModel.host = viperSettings.host
viperSettingsModel.requestLimit = viperSettings.requestLimit
viperSettingsModel.fetchMetadata = viperSettings.fetchMetadata
proxies.addAll(runBlocking { settingsController.getProxies() })
with(root) {
form {
fieldset {
field("Viper domain") {
combobox(viperSettingsModel.hostProperty, proxies)
}
field("Rate limit (requests/s)") {
add(Spinner<Int>(1, 5, viperSettingsModel.requestLimit.toInt()).apply {
valueProperty().onChange {
viperSettingsModel.requestLimit = it!!.toLong()
}
field("Rate limit (requests/s)") {
add(Spinner<Int>(1, 5, viperGirlsSettings.requestLimit.toInt()).apply {
viperSettingsModel.requestLimitProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Fetch Metadata") {
checkbox {
bind(viperSettingsModel.fetchMetadataProperty)
}
}
field("Authentication") {
add(ToggleSwitch().apply {
isSelected = viperGirlsSettings.login
viperSettingsModel.loginProperty.bind(selectedProperty())
})
}
fieldset {
visibleWhen(viperSettingsModel.loginProperty)
field("Username") {
textfield(viperSettingsModel.usernameProperty)
}
field("Password") {
passwordfield(viperSettingsModel.passwordProperty)
}
field("Leave likes") {
checkbox {
bind(viperSettingsModel.thanksProperty)
}
}
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Fetch Metadata") {
checkbox {
bind(viperSettingsModel.fetchMetadataProperty)
}
}
field("Authentication") {
add(ToggleSwitch().apply {
isSelected = viperSettings.login
viperSettingsModel.loginProperty.bind(selectedProperty())
})
}
fieldset {
visibleWhen(viperSettingsModel.loginProperty)
field("Username") {
textfield(viperSettingsModel.usernameProperty)
}
field("Password") {
passwordfield(viperSettingsModel.passwordProperty)
}
field("Leave likes") {
checkbox {
bind(viperSettingsModel.thanksProperty)
}
}
}
@@ -73,8 +70,4 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
}
}
}
override fun onUndock() {
coroutineScope.cancel()
}
}
@@ -12,6 +12,7 @@ import me.vripper.gui.components.fragments.AddLinksFragment
import me.vripper.gui.components.fragments.SettingsFragment
import me.vripper.gui.controller.ActionBarController
import me.vripper.gui.controller.PostController
import me.vripper.gui.controller.SettingsController
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
@@ -21,6 +22,7 @@ class ActionBarView : View() {
private val postController: PostController by inject()
private val postsTableView: PostsTableView by inject()
private val actionBarController: ActionBarController by inject()
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val running = SimpleIntegerProperty(0)
@@ -28,7 +30,7 @@ class ActionBarView : View() {
init {
with(root) {
id = "action_toolbar"
this.id = "action_toolbar"
padding = insets(all = 5)
button("Add links", FontIcon.of(Feather.PLUS)) {
contentDisplay = ContentDisplay.GRAPHIC_ONLY
@@ -79,7 +81,7 @@ class ActionBarView : View() {
coroutineScope.launch {
val clearPosts = async { postController.clearPosts() }.await()
runLater {
postsTableView.tableView.items.removeIf { clearPosts.contains(it.postId) }
postsTableView.tableView.items.removeIf { clearPosts.contains(it.vgPostId) }
}
}
}
@@ -90,10 +92,25 @@ class ActionBarView : View() {
contentDisplay = ContentDisplay.GRAPHIC_ONLY
tooltip("Open settings menu [Ctrl+P]")
action {
find<SettingsFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
coroutineScope.launch {
val downloadSettings = settingsController.findDownloadSettings()
val connectionSettings = settingsController.findConnectionSettings()
val viperGirlsSettings = settingsController.findViperGirlsSettings()
val systemSettings = settingsController.findSystemSettings()
val hostSettings = settingsController.findHostSettings()
runLater {
find<SettingsFragment>(
mapOf(
SettingsFragment::downloadSettings to downloadSettings,
SettingsFragment::connectionSettings to connectionSettings,
SettingsFragment::viperSettings to viperGirlsSettings,
SettingsFragment::systemSettings to systemSettings,
SettingsFragment::hostSettings to hostSettings,
)
).openModal()
}
}
}
}
separator(Orientation.VERTICAL)
@@ -1,16 +1,20 @@
package me.vripper.gui.components.views
import atlantafx.base.theme.CupertinoDark
import atlantafx.base.theme.CupertinoLight
import javafx.application.Application
import kotlinx.coroutines.runBlocking
import atlantafx.base.theme.*
import javafx.application.Application.setUserAgentStylesheet
import javafx.event.EventHandler
import javafx.scene.input.TransferMode
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.gui.controller.AppController
import me.vripper.gui.controller.MainController
import me.vripper.gui.controller.WidgetsController
import me.vripper.gui.event.GuiEventBus
import tornadofx.View
import tornadofx.onChange
import tornadofx.runLater
import tornadofx.vbox
import java.net.URI
class AppView : View() {
@@ -21,6 +25,8 @@ class AppView : View() {
private val statusBarView: StatusBarView by inject()
private val actionBarView: ActionBarView by inject()
private val widgetsController: WidgetsController by inject()
private val appController: AppController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
init {
title = "VRipper ${mainController.version}"
with(root) {
@@ -34,6 +40,24 @@ class AppView : View() {
}
prefWidth = widgetsController.currentSettings.width
prefHeight = widgetsController.currentSettings.height
onDragOver = EventHandler { event ->
if (event.dragboard.hasString()) {
event.acceptTransferModes(TransferMode.COPY)
}
event.consume()
}
onDragDropped = EventHandler { event ->
val dragBoard = event.dragboard
if (dragBoard != null && dragBoard.hasString()) {
val url = dragBoard.string.trim()
runCatching {
val uri = URI(url)
coroutineScope.launch {
appController.scan(uri.toString())
}
}
}
}
}
widgetsController.currentSettings.visibleToolbarPanelProperty.onChange { it ->
@@ -52,23 +76,16 @@ class AppView : View() {
}
}
widgetsController.currentSettings.darkModeProperty.onChange {
if (it) {
Application.setUserAgentStylesheet(CupertinoDark().userAgentStylesheet)
} else {
Application.setUserAgentStylesheet(CupertinoLight().userAgentStylesheet)
}
}
runLater {
if (widgetsController.currentSettings.localSession) {
runBlocking {
GuiEventBus.publishEvent(GuiEventBus.LocalSession)
}
} else {
runBlocking {
GuiEventBus.publishEvent(GuiEventBus.RemoteSession)
}
widgetsController.currentSettings.themeProperty.onChange {
when (it) {
"CupertinoLight" -> setUserAgentStylesheet(CupertinoLight().userAgentStylesheet)
"CupertinoDark" -> setUserAgentStylesheet(CupertinoDark().userAgentStylesheet)
"NordLight" -> setUserAgentStylesheet(NordLight().userAgentStylesheet)
"NordDark" -> setUserAgentStylesheet(NordDark().userAgentStylesheet)
"PrimerLight" -> setUserAgentStylesheet(PrimerLight().userAgentStylesheet)
"PrimerDark" -> setUserAgentStylesheet(PrimerDark().userAgentStylesheet)
"Dracula" -> setUserAgentStylesheet(Dracula().userAgentStylesheet)
else -> setUserAgentStylesheet(CupertinoLight().userAgentStylesheet)
}
}
}
@@ -25,19 +25,17 @@ import me.vripper.gui.utils.openLink
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
import kotlin.io.path.Path
class ImagesTableView : View("Photos") {
override val root = vbox(alignment = Pos.CENTER_RIGHT) {}
private val tableView: TableView<ImageModel>
private val imageController: ImageController by inject()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val items: ObservableList<ImageModel> = FXCollections.observableArrayList()
private var preview: Preview? = null
val jobs = mutableListOf<Job>()
override val root = vbox(alignment = Pos.CENTER_RIGHT) {}
private val preview: Preview = Preview(currentStage!!)
init {
with(root) {
@@ -73,25 +71,21 @@ class ImagesTableView : View("Photos") {
}
}
cellFactory = Callback {
val cell = PreviewTableCell<ImageModel>()
val cell = PreviewTableCell<ImageModel, String>()
cell.onMouseExited = EventHandler {
preview?.hide()
preview.cleanup()
}
cell.onMouseMoved = EventHandler {
preview?.previewPopup?.apply {
preview.previewPopup.apply {
x = it.screenX + 20
y = it.screenY + 10
}
}
cell.onMouseEntered = EventHandler { mouseEvent ->
preview?.hide()
preview.cleanup()
if (cell.tableRow.item != null && cell.tableRow.item.thumbUrl.isNotEmpty()) {
preview = Preview(
currentStage!!,
cell.tableRow.item.thumbUrl,
Path(widgetsController.currentSettings.cachePath)
)
preview?.previewPopup?.apply {
preview.display(cell.tableRow.item.postEntityId, listOf(cell.tableRow.item.thumbUrl))
preview.previewPopup.apply {
x = mouseEvent.screenX + 20
y = mouseEvent.screenY + 10
}
@@ -241,7 +235,7 @@ class ImagesTableView : View("Photos") {
modalStage?.width = 550.0
}
fun setPostId(postId: Long?) {
fun setPostId(id: Long?) {
runBlocking {
jobs.forEach { it.cancelAndJoin() }
jobs.clear()
@@ -249,11 +243,11 @@ class ImagesTableView : View("Photos") {
runLater {
items.clear()
}
if (postId == null) {
if (id == null) {
return
}
coroutineScope.launch {
val list = imageController.findImages(postId)
val list = imageController.findImages(id)
runLater {
items.addAll(list)
tableView.sort()
@@ -262,7 +256,7 @@ class ImagesTableView : View("Photos") {
}
coroutineScope.launch {
imageController.onUpdateImages(postId).collect { image ->
imageController.onUpdateImages(id).collect { image ->
runLater {
val imageModel = items.find { it.id == image.id } ?: return@runLater
@@ -1,9 +1,15 @@
package me.vripper.gui.components.views
import javafx.beans.property.SimpleStringProperty
import javafx.scene.control.ProgressIndicator.INDETERMINATE_PROGRESS
import javafx.scene.effect.DropShadow
import javafx.scene.input.KeyCode
import javafx.scene.input.KeyCodeCombination
import javafx.scene.input.KeyCombination
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import me.vripper.gui.VripperGuiApplication
import me.vripper.gui.components.fragments.SessionFragment
import me.vripper.gui.controller.WidgetsController
import me.vripper.gui.event.GuiEventBus
import me.vripper.gui.services.GrpcEndpointService
@@ -12,6 +18,8 @@ import me.vripper.gui.utils.ClipboardManager
import me.vripper.gui.utils.Watcher
import me.vripper.listeners.AppManager
import me.vripper.utilities.DatabaseManager
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
class LoadingView : View("VRipper") {
@@ -19,28 +27,49 @@ class LoadingView : View("VRipper") {
private val widgetsController: WidgetsController by inject()
private val grpcEndpointService: GrpcEndpointService by di("remoteAppEndpointService")
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val message = SimpleStringProperty("")
override val root = borderpane {}
init {
DatabaseManager.connect()
ClipboardManager.init()
coroutineScope.launch {
GuiEventBus.events.filterIsInstance(GuiEventBus.ApplicationInitialized::class).collect {
if (widgetsController.currentSettings.localSession) {
AppEndpointManager.set(GuiEventBus.LocalSession)
AppManager.start()
} else {
AppEndpointManager.set(GuiEventBus.RemoteSession)
runLater {
ClipboardManager.init()
}
message.set("")
AppManager.stop()
grpcEndpointService.disconnect()
if (!widgetsController.currentSettings.localSession) {
grpcEndpointService.connect(
widgetsController.currentSettings.remoteSessionModel.host,
widgetsController.currentSettings.remoteSessionModel.port,
widgetsController.currentSettings.remoteSessionModel.passcode,
)
val check = grpcEndpointService.versionCheck()
if (check == null) {
message.set("Unable to connect to ${widgetsController.currentSettings.remoteSessionModel.host}:${widgetsController.currentSettings.remoteSessionModel.port}")
return@collect
} else if (!check) {
message.set("Version Mismatch, client must be >= 6.6.0")
return@collect
}
}
runLater {
val sessionType = if (widgetsController.currentSettings.localSession) {
AppManager.start()
GuiEventBus.LocalSession
} else {
GuiEventBus.RemoteSession
}
AppEndpointManager.set(sessionType)
replaceWith(find<AppView>())
runBlocking {
GuiEventBus.publishEvent(sessionType)
}
}
if (it.args.isNotEmpty()) {
Watcher.notify(it.args[0])
}
@@ -49,16 +78,35 @@ class LoadingView : View("VRipper") {
with(root) {
padding = insets(all = 5)
top {
menubar {
menu("File") {
item("Change session", KeyCodeCombination(KeyCode.S, KeyCombination.SHIFT_DOWN)) {
graphic = FontIcon.of(Feather.LINK_2)
action {
find<SessionFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
}
separator()
item("Exit", KeyCodeCombination(KeyCode.X, KeyCombination.CONTROL_DOWN)).apply {
graphic = FontIcon.of(Feather.X_SQUARE)
action {
VripperGuiApplication.APP_INSTANCE.stop()
}
}
}
}.visibleWhen { message.isNotEmpty }
}
center {
progressindicator {
progress = INDETERMINATE_PROGRESS
}
}.visibleWhen { message.isEmpty }
text(message).visibleWhen { message.isNotEmpty }
}
effect = DropShadow()
}
}
override fun onUndock() {
coroutineScope.cancel()
}
}
@@ -153,39 +153,9 @@ class LogTableView : View() {
tableView.placeholder = Label("Loading")
tableView.sortOrder.add(tableView.columns.first { it.id == "time" })
coroutineScope.launch {
launch {
GuiEventBus.events.collect {
when (it) {
GuiEventBus.LocalSession, GuiEventBus.RemoteSession -> {
while (isActive) {
val result = runCatching { logController.getMaxEventLog() }
if (result.isSuccess) {
maxLogEvent = result.getOrNull()!!
break
}
}
while (isActive) {
val result = runCatching { logController.initLogger() }
if (result.isSuccess) {
break
}
}
}
GuiEventBus.ChangingSession -> runLater {
items.clear()
tableView.placeholder = Label("Loading")
}
else -> {}
}
}
}
launch {
logController.newLogs.collect {
logController.newLogs.let { flow ->
coroutineScope.launch {
flow.collect {
runLater {
items.sortWith(Comparator.comparing { it.sequence })
while (items.isNotEmpty() && (items.size >= maxLogEvent)) {
@@ -196,13 +166,49 @@ class LogTableView : View() {
}
}
}
}
launch {
logController.updateSettings.collect {
logController.updateSettings.let { flow ->
coroutineScope.launch {
flow.collect {
maxLogEvent = it.systemSettings.maxEventLog
}
}
}
coroutineScope.launch {
GuiEventBus.events.collect {
when (it) {
GuiEventBus.LocalSession, GuiEventBus.RemoteSession -> {
while (isActive) {
val result = runCatching { logController.getMaxEventLog() }
if (result.isSuccess) {
maxLogEvent = result.getOrNull()!!
break
}
}
while (isActive) {
val result = runCatching { logController.initLogger() }
if (result.isSuccess) {
break
}
}
}
GuiEventBus.ChangingSession -> runLater {
items.clear()
tableView.placeholder = Label("Loading")
}
is GuiEventBus.RemoteError -> runLater {
items.clear()
tableView.placeholder = Label("Connection Failure")
}
else -> {}
}
}
}
}
private fun openLog(item: LogModel) {
@@ -3,6 +3,7 @@ package me.vripper.gui.components.views
import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleIntegerProperty
import javafx.scene.control.ButtonType
import javafx.scene.control.ToggleGroup
import javafx.scene.input.KeyCode
import javafx.scene.input.KeyCodeCombination
import javafx.scene.input.KeyCombination
@@ -14,6 +15,7 @@ import me.vripper.gui.components.fragments.SessionFragment
import me.vripper.gui.components.fragments.SettingsFragment
import me.vripper.gui.controller.ActionBarController
import me.vripper.gui.controller.PostController
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.controller.WidgetsController
import me.vripper.gui.utils.openLink
import me.vripper.services.IAppEndpointService
@@ -29,6 +31,7 @@ class MenuBarView : View() {
private val widgetsController: WidgetsController by inject()
private val postController: PostController by inject()
private val actionBarController: ActionBarController by inject()
private val settingsController: SettingsController by inject()
private lateinit var appEndpointService: IAppEndpointService
private val running = SimpleIntegerProperty(0)
@@ -81,7 +84,7 @@ class MenuBarView : View() {
coroutineScope.launch {
val clearPosts = async { postController.clearPosts() }.await()
runLater {
postsTableView.tableView.items.removeIf { clearPosts.contains(it.postId) }
postsTableView.tableView.items.removeIf { clearPosts.contains(it.vgPostId) }
}
}
}
@@ -91,9 +94,23 @@ class MenuBarView : View() {
item("Settings", KeyCodeCombination(KeyCode.P, KeyCombination.CONTROL_DOWN)).apply {
graphic = FontIcon.of(Feather.SETTINGS)
action {
find<SettingsFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
coroutineScope.launch {
val downloadSettings = settingsController.findDownloadSettings()
val connectionSettings = settingsController.findConnectionSettings()
val viperGirlsSettings = settingsController.findViperGirlsSettings()
val systemSettings = settingsController.findSystemSettings()
val hostSettings = settingsController.findHostSettings()
runLater {
find<SettingsFragment>(
mapOf(
SettingsFragment::downloadSettings to downloadSettings,
SettingsFragment::connectionSettings to connectionSettings,
SettingsFragment::viperSettings to viperGirlsSettings,
SettingsFragment::systemSettings to systemSettings,
SettingsFragment::hostSettings to hostSettings,
)
).openModal()
}
}
}
}
@@ -125,9 +142,64 @@ class MenuBarView : View() {
checkmenuitem(
"Status Bar", KeyCodeCombination(KeyCode.F8)
).bind(widgetsController.currentSettings.visibleStatusBarPanelProperty)
checkmenuitem(
"Dark mode"
).bind(widgetsController.currentSettings.darkModeProperty)
menu("Theme") {
val toggleGroup = ToggleGroup()
val cupertinoLight = radiomenuitem("Cupertino Light", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "CupertinoLight"
widgetsController.currentSettings.darkMode = false
}
}
val cupertinoDark = radiomenuitem("Cupertino Dark", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "CupertinoDark"
widgetsController.currentSettings.darkMode = true
}
}
val nordLight = radiomenuitem("Nord Light", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "NordLight"
widgetsController.currentSettings.darkMode = false
}
}
val nordDark = radiomenuitem("Nord Dark", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "NordDark"
widgetsController.currentSettings.darkMode = true
}
}
val primerLight = radiomenuitem("Primer Light", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "PrimerLight"
widgetsController.currentSettings.darkMode = false
}
}
val primerDark = radiomenuitem("Primer Dark", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "PrimerDark"
widgetsController.currentSettings.darkMode = true
}
}
val dracula = radiomenuitem("Dracula", toggleGroup).apply {
this.selectedProperty().onChange {
widgetsController.currentSettings.theme = "Dracula"
widgetsController.currentSettings.darkMode = true
}
}
when (widgetsController.currentSettings.theme) {
"CupertinoLight" -> cupertinoLight.isSelected = true
"CupertinoDark" -> cupertinoDark.isSelected = true
"NordLight" -> nordLight.isSelected = true
"NordDark" -> nordDark.isSelected = true
"PrimerLight" -> primerLight.isSelected = true
"PrimerDark" -> primerDark.isSelected = true
"Dracula" -> dracula.isSelected = true
else -> cupertinoLight.isSelected = true
}
}
}
menu("Help") {
item("Database migration").apply {
@@ -199,13 +271,15 @@ class MenuBarView : View() {
}
}
downloadActiveProperty.bind(running.greaterThan(0))
coroutineScope.launch {
actionBarController.onQueueStateUpdate.collect {
runLater {
running.set(it.running)
actionBarController.onQueueStateUpdate.let { flow ->
coroutineScope.launch {
flow.collect {
runLater {
running.set(it.running)
}
}
}
}
}
}
@@ -18,14 +18,14 @@ class PostInfoView : View() {
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val imagesTableView: ImagesTableView by inject()
private val postModel: PostModel = PostModel(
-1, "", 0.0, "", "", 0, 0, "", "", -1, "", "", "", emptyList(), emptyList(), "", 0
-1, -1, "", 0.0, "", "", 0, 0, "", "", -1, "", "", "", emptyList(), emptyList(), "", 0
)
override val root = tabpane()
init {
with(root) {
id = "postinfo_panel"
this.id = "postinfo_panel"
tabClosingPolicy = TabPane.TabClosingPolicy.UNAVAILABLE
tab("General") {
graphic = FontIcon(Feather.INFO)
@@ -77,11 +77,10 @@ class PostInfoView : View() {
}
}
fun setPostId(postId: Long?) {
imagesTableView.setPostId(postId)
if (postId == null) {
fun setPostId(id: Long?) {
imagesTableView.setPostId(id)
if (id == null) {
postModel.apply {
this.postId = -1
this.title = ""
this.progress = 0.0
this.status = ""
@@ -101,13 +100,15 @@ class PostInfoView : View() {
return
}
coroutineScope.launch {
val model: PostModel? = postController.find(postId)
val model: PostModel? = postController.find(id)
if (model == null) {
return@launch
}
runLater {
postModel.apply {
this.postId = model.postId
this.id = model.id
this.vgPostId = model.vgPostId
this.vgThreadId = model.vgThreadId
this.title = model.title
this.progress = model.progress
this.status = model.status.lowercase().replaceFirstChar { it.uppercase() }
@@ -126,33 +127,37 @@ class PostInfoView : View() {
}
}
}
coroutineScope.launch {
postController.updatePostsFlow.filter {
it.postId == postModel.postId
}.collect { post ->
runLater {
postModel.status = post.status.stringValue.lowercase().replaceFirstChar { it.uppercase() }
postModel.progressCount = postController.progressCount(
post.total, post.done, post.downloaded
)
postModel.order = post.rank + 1
postModel.done = post.done
postModel.progress = postController.progress(
post.total, post.done
)
postModel.path = post.getDownloadFolder()
postModel.folderName = post.folderName
postController.updatePostsFlow.let { flow ->
coroutineScope.launch {
flow.filter {
it.id == postModel.id
}.collect { post ->
runLater {
postModel.status = post.status.stringValue.lowercase().replaceFirstChar { it.uppercase() }
postModel.progressCount = postController.progressCount(
post.total, post.done, post.downloaded
)
postModel.done = post.done
postModel.progress = postController.progress(
post.total, post.done
)
postModel.path = post.getDownloadFolder()
postModel.folderName = post.folderName
}
}
}
}
coroutineScope.launch {
postController.updateMetadataFlow.filter {
it.postId == postModel.postId
}.collect {
runLater {
postModel.altTitles = FXCollections.observableArrayList(it.data.resolvedNames)
postModel.postedBy = it.data.postedBy
postController.updateMetadataFlow.let { flow ->
coroutineScope.launch {
flow.filter {
it.postIdRef == postModel.id
}.collect {
runLater {
postModel.altTitles = FXCollections.observableArrayList(it.data.resolvedNames)
postModel.postedBy = it.data.postedBy
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More