Compare commits

...
70 Commits
Author SHA1 Message Date
dev-claw 4ba422eee1 Update version 2026-04-18 01:39:04 +01:00
dev-claw e80d704b03 fixes #295
fixes #294
fixes #293
fixes #291
fixes #286
fixes #285
2026-04-18 01:37:49 +01:00
EVgZcQvc8iandGitHub 562c36bf28 Fix open file directory on Linux when the path contains spaces (#282) 2026-04-15 22:09:15 +01:00
dev-claw c3d07a8503 Update release.yml 2025-12-23 14:19:38 +01:00
dev-claw 30c9c9f24d Update version 2025-12-23 14:13:02 +01:00
dev-claw ecf74b7b00 bug fixes 2025-12-23 14:11:22 +01:00
dev-claw ed0c5f76a6 bug fixes 2025-10-12 14:08:00 +01:00
dev-claw 46ddb66dc7 bug fixes 2025-10-11 22:35:40 +01:00
dev-claw 11c9b665ce Update version 2025-10-11 17:39:25 +01:00
dev-claw 5b17f17e18 Update package.yml 2025-10-11 17:35:57 +01:00
dev-claw d84f19b4af Create package.yml 2025-10-11 17:20:49 +01:00
dev-claw 12fad45e8b remove release-manual.yml 2025-10-11 17:01:00 +01:00
dev-claw dc74e1573c bug fixes 2025-10-11 16:22:48 +01:00
dev-clawandGitHub f9b89c0b4d Update package.yml 2025-10-11 16:14:02 +01:00
dev-clawandGitHub 1bf75f0e8e Update release.yml 2025-10-10 21:51:44 +01:00
dev-clawandGitHub c13a9f93b3 Update release.yml (#278) 2025-10-10 21:49:32 +01:00
dev-claw 3302ceda20 Update release.yml & package.yml 2025-10-10 21:37:57 +01:00
dev-claw 3adf5a3254 Update version 2025-10-10 21:22:22 +01:00
dev-clawandGitHub ba22747384 fixes #274 (#277)
fixes #274
fixes #275
fixes #276
2025-10-10 21:06:04 +01:00
dev-claw 455833377d Update package.yml & release.yml 2025-10-10 21:02:32 +01:00
dev-claw 27be6e7167 Update package.yml 2025-10-10 20:41:45 +01:00
dev-clawandGitHub ab54aae48e Update README.md 2025-09-30 15:37:51 +01:00
dev-claw 34b31a51a5 version bump 2025-09-26 12:15:37 +01:00
dev-clawandGitHub 99a8aa0ce0 fixes #267 (#269) 2025-09-26 12:10:50 +01:00
dev-claw bdb67475a6 update README.md 2025-09-19 21:15:25 +01:00
dev-claw 24c961644e update README.md 2025-09-19 21:07:39 +01:00
dev-claw 3c3b79f581 update README.md 2025-09-19 20:22:09 +01:00
dev-claw 8c30e5a02d fixes #239 2025-09-19 20:09:05 +01:00
dev-claw 398220d518 update README.md 2025-09-19 20:04:54 +01:00
dev-clawandGitHub af9a439d81 fixes #239 (#266) 2025-09-19 19:13:18 +01:00
dev-claw c9760511b5 version bump 2025-09-19 19:03:27 +01:00
dev-clawandGitHub 59fe4d227f fixes #239 (#265) 2025-09-19 19:02:36 +01:00
dev-clawandGitHub 6df84c0756 fixes #261 (#264) 2025-09-19 18:41:04 +01:00
dev-clawandGitHub aa0119f881 fixes #239 (#263) 2025-09-19 18:29:03 +01:00
dev-clawandGitHub 9283ab7837 fixes #260 (#262) 2025-09-06 14:19:34 +01:00
dev-claw ca77074ef6 version bump 2025-08-19 15:42:07 +01:00
dev-clawandGitHub 3cb251cb36 fixes #250 (#252) 2025-08-19 15:38:08 +01:00
dev-clawandGitHub 73f478c1bc fixes #248 (#249) 2025-08-18 19:46:05 +01:00
dev-claw bb1dfbfee1 version bump 2025-08-18 19:36:51 +01:00
dev-clawandGitHub a89fe8571c fixes #243 (#246) 2025-08-18 19:34:21 +01:00
dev-clawandGitHub 44c8e443f4 fixes #240 (#241) 2025-08-08 15:41:43 +01:00
dev-claw 7ddc5d00b9 fix counter 2025-08-07 15:19:41 +01:00
dev-claw 00039e230b version bump 2025-08-07 14:56:30 +01:00
dev-clawandGitHub f4821aafac fixes #237 (#238) 2025-08-07 14:55:03 +01:00
dev-clawandGitHub d7c8377a71 fixes #231 (#232) 2025-04-15 00:05:27 +01:00
dev-claw 1194faec28 version bump 2025-04-13 19:44:40 +01:00
dev-clawandGitHub 4b789bac8f fixes #228 (#229) 2025-04-13 19:43:28 +01:00
dev-claw 899e6d2893 version bump 2025-03-22 22:52:14 +01:00
dev-clawandGitHub 4c96e79802 fixes #225 (#226) 2025-03-22 22:18:59 +01:00
dev-claw 4a001efa9f version bump 2025-03-21 16:41:49 +01:00
dev-clawandGitHub 7035b1cce0 fixes #223 (#224) 2025-03-21 16:40:38 +01:00
dev-clawandGitHub b9862a5042 fixes #221 (#222) 2025-03-11 16:08:48 +01:00
dev-claw 8bcdb80149 version bump 2025-03-11 16:05:08 +01:00
dev-clawandGitHub e44220321e fixes #219 (#220) 2025-03-11 15:42:41 +01:00
dev-claw f2c9806665 version bump 2025-03-04 18:03:18 +01:00
dev-clawandGitHub fd586a522c fixes #216 (#218) 2025-03-04 18:00:16 +01:00
dev-clawandGitHub 6be744a7a7 Update release.yml 2025-03-02 14:11:22 +01:00
dev-clawandGitHub 6228551836 Update README.md 2024-12-29 12:48:07 +01:00
dev-claw b872d5a1a5 update links 2024-12-27 19:27:28 +01:00
dev-claw 61ce16d219 version bump 2024-12-27 19:14:46 +01:00
dev-clawandGitHub c74d711702 fixes #208 (#209) 2024-12-27 19:12:07 +01:00
death-clawandGitHub 095e69d3c9 Update README.md 2024-12-27 11:45:42 +01:00
death-clawandGitHub 60bee4df2c Update README.md 2024-12-27 11:39:59 +01:00
death-clawandGitHub 3d68691037 Update README.md 2024-12-26 22:46:03 +01:00
death-claw 4456d3d797 version bump 2024-12-25 16:24:46 +01:00
death-clawandGitHub 2bcdd81eeb fixes #206 (#207) 2024-12-25 16:23:13 +01:00
death-claw 9652c7725e version bump 2024-12-21 17:59:27 +01:00
death-clawandGitHub 01ed204d12 Update README.md 2024-12-21 17:47:27 +01:00
death-clawandGitHub b8639be241 fixes #204 (#205) 2024-12-21 17:44:28 +01:00
death-clawandGitHub b429c58eb5 version bump (#203) 2024-12-14 18:01:02 +01:00
268 changed files with 20896 additions and 15114 deletions
+174 -37
View File
@@ -2,54 +2,191 @@ name: Package
on:
workflow_dispatch:
inputs:
release:
description: 'Release'
required: true
type: string
jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ]
runs-on: ${{ matrix.os }}
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
settings-path: ${{ github.workspace }} # location for the settings.xml file
- name: Set up jdk25
uses: actions/setup-java@v3
with:
java-version: '25'
distribution: 'zulu'
- name: Build with Maven
run: mvn -B install --file pom.xml
- name: Build gui jar
run: |
mvn -N -q install --file pom.xml
mvn -B -q install --file vripper-core/pom.xml
mvn -B -q install --file vripper-gui/pom.xml
mv vripper-gui/target/vripper-gui-${{ inputs.release }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ inputs.release }}.jar
- name: Rename GUI artifact
run: mv vripper-gui/target/vripper-gui-*.jar vripper-gui/target/vripper-gui.jar
- if: matrix.os == 'ubuntu-latest'
name: Release gui Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-gui/target/vripper-noarch-gui-${{ inputs.release }}.jar
# Start building WEB jar in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Build web Jar
run: |
mvn -B -q install --file vripper-web-ui/pom.xml
mvn -B -q install --file vripper-web/pom.xml
mv vripper-web/target/vripper-web-${{ inputs.release }}.jar vripper-web/target/vripper-noarch-web-${{ inputs.release }}.jar
- name: Rename WEB artifact
run: mv vripper-web/target/vripper-web-*.jar vripper-web/target/vripper-web.jar
- if: matrix.os == 'ubuntu-latest'
name: Release web Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-web/target/vripper-noarch-web-${{ inputs.release }}.jar
# End building WEB jar in ubuntu only
# Start building docker image in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- if: matrix.os == 'ubuntu-latest'
name: Package Linux binaries
run: |
cd jpackage
cp ../vripper-gui/target/vripper-gui.jar jar/vripper-gui.jar
jpackage --type deb "@jpackage.cfg" "@jpackage-linux.cfg"
jpackage --type rpm "@jpackage.cfg" "@jpackage-linux.cfg"
ls -la dist
- if: matrix.os == 'ubuntu-latest'
name: Build Docker image
run: |
mkdir docker_build && cp Dockerfile docker_build/Dockerfile && cp vripper-web/target/vripper-noarch-web-${{ inputs.release }}.jar docker_build/vripper-web.jar
docker build -t ghcr.io/dev-claw/vripper-web docker_build
docker tag ghcr.io/dev-claw/vripper-web:latest ghcr.io/dev-claw/vripper-web:${{ inputs.release }}
docker push ghcr.io/dev-claw/vripper-web:latest
docker push ghcr.io/dev-claw/vripper-web:${{ inputs.release }}
# End building docker image in ubuntu only
- if: matrix.os == 'windows-latest'
name: Package Windows binaries
run: |
cd jpackage
cp ../vripper-gui/target/vripper-gui.jar jar/vripper-gui.jar
jpackage "@jpackage.cfg" "@jpackage-windows.cfg"
dir dist
- name: Prepare Packaging
run: |
cp vripper-gui/target/vripper-noarch-gui-${{ inputs.release }}.jar jpackage/jar/vripper-gui.jar
- if: matrix.os == 'macos-latest'
name: Package macOS binaries
run: |
cd jpackage
cp ../vripper-gui/target/vripper-gui.jar jar/vripper-gui.jar
jpackage "@jpackage.cfg" "@jpackage-macos.cfg"
ls -la dist
- if: matrix.os == 'ubuntu-latest'
name: Package for Linux
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.png
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type deb
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type rpm
mv dist/vripper-${{ inputs.release }}-1.x86_64.rpm dist/vripper-linux-${{ inputs.release }}.x86_64.rpm
mv dist/vripper_${{ inputs.release }}-1_amd64.deb dist/vripper-linux-${{ inputs.release }}_amd64.deb
- if: matrix.os == 'windows-latest'
name: Package for Windows
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.ico
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type msi
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type exe
cd dist
ren VRipper-${{ inputs.release }}.msi vripper-windows-installer-${{ inputs.release }}.msi
ren VRipper-${{ inputs.release }}.exe vripper-windows-installer-${{ inputs.release }}.exe
- if: matrix.os == 'macos-latest'
name: Package for macOS(arm64)
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ inputs.release }}.pkg dist/vripper-macos-${{ inputs.release }}.arm64.pkg
mv dist/VRipper-${{ inputs.release }}.dmg dist/vripper-macos-${{ inputs.release }}.arm64.dmg
- if: matrix.os == 'macos-13'
name: Package for macOS(x86_64)
run: |
cd jpackage
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ inputs.release }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ inputs.release }}.pkg dist/vripper-macos-${{ inputs.release }}.x86_64.pkg
mv dist/VRipper-${{ inputs.release }}.dmg dist/vripper-macos-${{ inputs.release }}.x86_64.dmg
- if: matrix.os == 'ubuntu-latest'
name: Zip Ubuntu portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper'
filename: 'vripper-linux-portable-${{ inputs.release }}.zip'
- if: matrix.os == 'windows-latest'
name: Zip Windows portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper'
filename: 'vripper-windows-portable-${{ inputs.release }}.zip'
- if: matrix.os == 'macos-latest'
name: Zip macOS(arm64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ inputs.release }}.arm64.zip'
- if: matrix.os == 'macos-13'
name: Zip macOS(x86_64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ inputs.release }}.x86_64.zip'
- if: matrix.os == 'ubuntu-latest'
name: Release packages for Linux
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-linux-${{ inputs.release }}.x86_64.rpm
jpackage/dist/vripper-linux-${{ inputs.release }}_amd64.deb
jpackage/dist/vripper-linux-portable-${{ inputs.release }}.zip
- if: matrix.os == 'windows-latest'
name: Release packages for Windows
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-windows-installer-${{ inputs.release }}.msi
jpackage/dist/vripper-windows-installer-${{ inputs.release }}.exe
jpackage/dist/vripper-windows-portable-${{ inputs.release }}.zip
- if: matrix.os == 'macos-latest'
name: Release packages for macOS(arm64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ inputs.release }}.arm64.pkg
jpackage/dist/vripper-macos-${{ inputs.release }}.arm64.dmg
jpackage/dist/vripper-macos-portable-${{ inputs.release }}.arm64.zip
- if: matrix.os == 'macos-13'
name: Release packages for macOS(x86_64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ inputs.release }}.x86_64.pkg
jpackage/dist/vripper-macos-${{ inputs.release }}.x86_64.dmg
jpackage/dist/vripper-macos-portable-${{ inputs.release }}.x86_64.zip
+86 -44
View File
@@ -8,89 +8,113 @@ jobs:
build:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest, macos-latest ]
os: [ ubuntu-latest, windows-latest, macos-15-intel, macos-latest ]
runs-on: ${{ matrix.os }}
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v3
- name: Set up JDK 21
- name: Set up jdk25
uses: actions/setup-java@v3
with:
java-version: '21'
distribution: 'temurin'
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
settings-path: ${{ github.workspace }} # location for the settings.xml file
java-version: '25'
distribution: 'zulu'
cache: maven
- name: Build parent project
run: mvn -N -q install --file pom.xml
- name: Build Core Jar
run: mvn -B -q install --file vripper-core/pom.xml
- name: Build GUI Jar
run: mvn -B -q install --file vripper-gui/pom.xml
- name: Rename GUI Jar
run: mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
- name: Build gui jar
run: |
mvn -N -q install --file pom.xml
mvn -B -q install --file vripper-core/pom.xml
mvn -B -q install --file vripper-gui/pom.xml
mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
- if: matrix.os == 'ubuntu-latest'
name: Release GUI Jar
name: Release gui Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
# Start building WEB jar in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Build WEB Jar
name: Build web Jar
run: |
mvn -B -q install --file vripper-web-ui/pom.xml
mvn -B -q install --file vripper-web/pom.xml
mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
- if: matrix.os == 'ubuntu-latest'
name: Rename WEB Jar
run: mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
- if: matrix.os == 'ubuntu-latest'
name: Release WEB Jar
name: Release web Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
# End building WEB jar in ubuntu only
# Start building docker image in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- if: matrix.os == 'ubuntu-latest'
name: Build Docker image
run: |
mkdir docker_build && cp Dockerfile docker_build/Dockerfile && cp vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar docker_build/vripper-web.jar
docker build -t ghcr.io/dev-claw/vripper-web docker_build
docker tag ghcr.io/dev-claw/vripper-web:latest ghcr.io/dev-claw/vripper-web:${{ github.event.release.tag_name }}
docker push ghcr.io/dev-claw/vripper-web:latest
docker push ghcr.io/dev-claw/vripper-web:${{ github.event.release.tag_name }}
# End building docker image in ubuntu only
- name: Prepare Packaging
run: |
cp vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar jpackage/jar/vripper-gui.jar
- if: matrix.os == 'ubuntu-latest'
name: Package Linux
name: Package for Linux
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.png
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --type deb
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --type rpm
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.png
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type deb
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type rpm
mv dist/vripper-${{ github.event.release.tag_name }}-1.x86_64.rpm dist/vripper-linux-${{ github.event.release.tag_name }}.x86_64.rpm
mv dist/vripper_${{ github.event.release.tag_name }}-1_amd64.deb dist/vripper-linux-${{ github.event.release.tag_name }}_amd64.deb
- if: matrix.os == 'windows-latest'
name: Package Windows
name: Package for Windows
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.ico
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --type msi
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --type exe
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.ico
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type msi
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type exe
cd dist
ren VRipper-${{ github.event.release.tag_name }}.msi vripper-windows-installer-${{ github.event.release.tag_name }}.msi
ren VRipper-${{ github.event.release.tag_name }}.exe vripper-windows-installer-${{ github.event.release.tag_name }}.exe
- if: matrix.os == 'macos-latest'
name: Package macOS
name: Package for macOS(arm64)
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
- if: matrix.os == 'macos-15-intel'
name: Package for macOS(x86_64)
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
- if: matrix.os == 'ubuntu-latest'
name: Zip Ubuntu portable
@@ -111,13 +135,22 @@ jobs:
filename: 'vripper-windows-portable-${{ github.event.release.tag_name }}.zip'
- if: matrix.os == 'macos-latest'
name: Zip MacOS portable
name: Zip macOS(arm64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.zip'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip'
- if: matrix.os == 'macos-15-intel'
name: Zip macOS(x86_64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip'
- if: matrix.os == 'ubuntu-latest'
name: Release packages for Linux
@@ -138,10 +171,19 @@ jobs:
jpackage/dist/vripper-windows-portable-${{ github.event.release.tag_name }}.zip
- if: matrix.os == 'macos-latest'
name: Release packages for macOS
name: Release packages for macOS(arm64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.zip
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip
- if: matrix.os == 'macos-15-intel'
name: Release packages for macOS(x86_64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip
+2 -1
View File
@@ -4,4 +4,5 @@
/**/*/build-dir
/**/*/java-runtime
.idea
.vripper
.vripper
.flattened-pom.xml
+4
View File
@@ -0,0 +1,4 @@
FROM azul/zulu-openjdk:25-latest
WORKDIR /opt/vripper-web
ENTRYPOINT ["java", "-jar", "vripper-web.jar"]
ADD vripper-web.jar /opt/vripper-web/vripper-web.jar
+119 -27
View File
@@ -1,26 +1,137 @@
# VRipper!
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to).
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to)
![GitHub Image](/image.png)
## Donation
To support this project, you can make a donation to its current maintainer
## Requirements
Direct access to `vipergirls.to` domain.
[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/devclaw)
Or with Cryptocurrency
ETH: 0xDdac82B16dC5E3D742fc915ffF583D8548A301cA
BTC: bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr
## Supported Image Hosts
The following hosts are supported:
* acidimg.cc
* imagetwist.com
* imagezilla.com
* imgspice.com
* imagebam.com
* imgbox.com
* imx.to
* pimpandhost.com
* pixhost.to
* pixxxels.cc
* turboimagehost.com
* postimg.cc
* imagevenue.com
* pixroute.to
* vipr.im
## Installing VRipper
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
:---:|:---:|:---:|:---:
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-windows-installer-5.8.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-windows-installer-5.8.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-windows-portable-5.8.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-macos-5.8.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-macos-5.8.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-macos-portable-5.8.0.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-linux-5.8.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-linux-5.8.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-linux-portable-5.8.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-noarch-gui-5.8.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-noarch-web-5.8.0.jar)
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
:---:|:---:|:---:|:---:|:---:
[Installer (EXE)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-windows-installer-6.5.8.exe) <br /> [Installer (MSI)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-windows-installer-6.5.8.msi) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-windows-portable-6.5.8.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-macos-6.5.8.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-macos-6.5.8.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-macos-portable-6.5.8.x86_64.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-macos-6.5.8.arm64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-macos-6.5.8.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-macos-portable-6.5.8.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-linux-6.5.3_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-linux-6.5.8.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-linux-portable-6.5.8.zip) | [Java GUI (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-noarch-gui-6.5.8.jar) <br /> [Java Web (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.8/vripper-noarch-web-6.5.8.jar)
Source code and previous versions are available on the [Releases page](https://github.com/death-claw/vripper-project/releases).
Source code and previous versions are available on
the [Releases page](https://github.com/dev-claw/vripper-project/releases).
Application data (application logs, settings and persisted data) is stored in:
* Windows --> `C:\USERS\<your Windows username>\vripper`
* Linux and macOS --> `HOME_FOLDER/.config/vripper`
## Docker Support
The project is supplied as a container image and can be run locally or in production using Docker. By default:
- **gRPC server/client** uses port **30000**
- **Web UI** runs on port **8080**
### Run with `docker run`
```bash
docker run -d \
-p 30000:30000 \
-p 8080:8080 \
-e GRPC_ENABLED=true \
-e GRPC_PASSPHRASE=my-secure-secret \
ghcr.io/dev-claw/vripper-web:latest
```
### Run with Docker Compose
```yaml
services:
app:
image: ghcr.io/dev-claw/vripper-web:latest
environment:
- GRPC_ENABLED=true
- GRPC_PASSPHRASE=super-secret-passphrase
ports:
- "30000:30000"
- "8080:8080"
```
> **Tip:** For production, prefer storing the passphrase as a secret (see `Security Notes` below) rather than writing it
> directly in the Compose file.
### Environment Variables
The container supports these environment variables for enabling and securing the gRPC server.
### `GRPC_ENABLED`
- **Type:** String (interpreted as Boolean)
- **Accepted values:** `true` (case-insensitive) to enable; any other value or unset → disabled
- **Default:** `false` (gRPC disabled)
- **Behavior:**
- When `GRPC_ENABLED=true`, the application will attempt to start the gRPC server (default port: `30000`).
- When absent or set to anything other than `true`, the gRPC server will not be started.
- **Example (shell):**
```bash
export GRPC_ENABLED=true
```
- **Example (Docker Compose):**
```yaml
environment:
- GRPC_ENABLED=true
```
- **Notes / Best practices:**
- Treat the value as a simple on/off toggle. To avoid ambiguity, set explicitly to `true` or `false`.
- If you enable gRPC, make sure any required authentication (see `GRPC_PASSPHRASE`) is also configured.
### `GRPC_PASSPHRASE`
- **Type:** String
- **Default:** *no default* (empty / unset)
- **Required if:** `GRPC_ENABLED=true`
- **Purpose:** Shared secret used to authenticate gRPC clients. The application should validate that the passphrase is
present when gRPC is enabled and use it to verify client connections.
- **Example (shell):**
```bash
export GRPC_PASSPHRASE="my-very-strong-passphrase"
```
## Default Ports
- **gRPC server/client:** `30000`
- **Web UI:** `8080`
You can map these ports when running the container. The application supports overriding these defaults with environment
variables (`GRPC_PORT`, `SERVER_PORT`), configure them as needed.
## Important Note About Proxies
The use of proxies within VRipper is worthless, please stop using them **for now**. You will get **403 error** codes.
@@ -29,24 +140,6 @@ The use of proxies within VRipper is worthless, please stop using them **for now
If your ISP is blocking access to `vipergirls.to` domain, consider using a VPN or [Cloudflare WARP](https://one.one.one.one/) to bypass the block.
## Supported Image Hosts
The following hosts are supported:
* acidimg.cc
* imagetwist.com
* imagezilla.com
* imgspice.com
* imagebam.com
* imgbox.com
* imx.to
* pimpandhost.com
* pixhost.to
* pixxxels.cc
* turboimagehost.com
* postimg.cc
* imagevenue.com
* pixroute.to
* vipr.im
## Instructions to run from Jar file
You need Java 21+, you can download from https://adoptium.net/
@@ -62,7 +155,6 @@ For the WEB app
Application data (application logs, settings and persisted data) is stored in the location where you launched the jar for both GUI and WEB
## How to build
You need JDK 21 and a recent version of maven 3.8.x+
-1
View File
@@ -1,4 +1,3 @@
--icon icon.png
--java-options "-Dvripper.portable=false"
--linux-package-name vripper
--linux-app-release 1
-1
View File
@@ -1,4 +1,3 @@
--icon icon.icns
--java-options "-Dvripper.portable=false"
--mac-package-identifier me.mnlr.vripper.vripper-gui
--mac-package-name VRipper
+1 -2
View File
@@ -1,8 +1,7 @@
--icon icon.ico
--java-options "-Dvripper.portable=false"
--win-dir-chooser
--win-menu
--win-per-user-install
--win-shortcut
--win-shortcut-prompt
--win-update-url https://github.com/death-claw/vripper-project/releases
--win-update-url https://github.com/dev-claw/vripper-project/releases
+1 -1
View File
@@ -5,4 +5,4 @@
--name VRipper
--description "Image ripper tool for vipergirls"
--dest dist
--vendor "death-claw"
--vendor "dev-claw"
+10
View File
@@ -0,0 +1,10 @@
[Desktop Entry]
Name=VRipper
Comment=Image ripper tool for vipergirls
Exec=/opt/vripper/bin/VRipper
Icon=/opt/vripper/lib/VRipper.png
Terminal=false
Type=Application
Categories=Utility
MimeType=
StartupWMClass=me.vripper.gui.VripperGuiApplication

Before

Width:  |  Height:  |  Size: 303 KiB

After

Width:  |  Height:  |  Size: 303 KiB

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

+40 -22
View File
@@ -8,34 +8,37 @@
<packaging>pom</packaging>
<version>${revision}</version>
<properties>
<java.version>21</java.version>
<java.version>25</java.version>
<maven.deploy.skip>false</maven.deploy.skip>
<timestamp>${maven.build.timestamp}</timestamp>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss</maven.build.timestamp.format>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<revision>5.8.0</revision>
<kotlin.version>1.9.22</kotlin.version>
<slf4j.version>2.0.12</slf4j.version>
<logback.version>1.5.0</logback.version>
<kotlinx-coroutines.version>1.8.0</kotlinx-coroutines.version>
<exposed.version>0.47.0</exposed.version>
<revision>6.7.0</revision>
<app-version>6.7.0</app-version>
<kotlin.version>2.2.20</kotlin.version>
<slf4j.version>2.0.16</slf4j.version>
<logback.version>1.5.12</logback.version>
<kotlinx-coroutines.version>1.10.2</kotlinx-coroutines.version>
<exposed.version>0.55.0</exposed.version>
<snakeyaml.version>2.0</snakeyaml.version>
<koin.version>3.5.3</koin.version>
<koin.version>4.0.0</koin.version>
<h2.version>2.2.224</h2.version>
<htmlcleaner.version>2.29</htmlcleaner.version>
<failsafe.version>2.4.4</failsafe.version>
<failsafe.version>3.3.2</failsafe.version>
<caffeine.version>3.1.8</caffeine.version>
<jna-platform.version>5.14.0</jna-platform.version>
<kotlinx-serialization-json.version>1.6.2</kotlinx-serialization-json.version>
<httpclient5.version>5.3.1</httpclient5.version>
<commons-codec.version>1.16.1</commons-codec.version>
<liquibase-core.version>4.26.0</liquibase-core.version>
<kotlinx-serialization-json.version>1.9.0</kotlinx-serialization-json.version>
<httpclient5.version>5.4</httpclient5.version>
<httpcore5.version>5.3</httpcore5.version>
<liquibase-core.version>4.29.2</liquibase-core.version>
<liquibase-slf4j.version>5.0.0</liquibase-slf4j.version>
<grpc.kotlin.version>1.4.1</grpc.kotlin.version>
<java.grpc.version>1.62.2</java.grpc.version>
<protobuf.version>3.25.3</protobuf.version>
<java.grpc.version>1.68.1</java.grpc.version>
<protobuf.version>3.25.5</protobuf.version>
<sqlite-jdbc.version>3.47.0.0</sqlite-jdbc.version>
<bucket4j_jdk17-core.version>8.14.0</bucket4j_jdk17-core.version>
</properties>
<build>
<plugins>
@@ -129,14 +132,19 @@
<groupId>com.h2database</groupId>
<version>${h2.version}</version>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>${sqlite-jdbc.version}</version>
</dependency>
<dependency>
<artifactId>htmlcleaner</artifactId>
<groupId>net.sourceforge.htmlcleaner</groupId>
<version>${htmlcleaner.version}</version>
</dependency>
<dependency>
<groupId>dev.failsafe</groupId>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<version>${failsafe.version}</version>
</dependency>
<dependency>
@@ -170,9 +178,14 @@
<version>${httpclient5.version}</version>
</dependency>
<dependency>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
<version>${commons-codec.version}</version>
<artifactId>httpcore5</artifactId>
<groupId>org.apache.httpcomponents.core5</groupId>
<version>${httpcore5.version}</version>
</dependency>
<dependency>
<artifactId>httpcore5-h2</artifactId>
<groupId>org.apache.httpcomponents.core5</groupId>
<version>${httpcore5.version}</version>
</dependency>
<dependency>
<artifactId>liquibase-core</artifactId>
@@ -194,15 +207,20 @@
<artifactId>grpc-protobuf</artifactId>
<version>${java.grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
<version>${java.grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-kotlin</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
<version>${java.grpc.version}</version>
<groupId>com.bucket4j</groupId>
<artifactId>bucket4j_jdk17-core</artifactId>
<version>${bucket4j_jdk17-core.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
+11 -7
View File
@@ -51,6 +51,10 @@
<artifactId>h2</artifactId>
<groupId>com.h2database</groupId>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
</dependency>
<dependency>
<artifactId>httpclient5</artifactId>
<groupId>org.apache.httpcomponents.client5</groupId>
@@ -65,16 +69,12 @@
</dependency>
<dependency>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<groupId>dev.failsafe</groupId>
</dependency>
<dependency>
<artifactId>caffeine</artifactId>
<groupId>com.github.ben-manes.caffeine</groupId>
</dependency>
<dependency>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna-platform</artifactId>
@@ -104,6 +104,10 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-kotlin</artifactId>
</dependency>
<dependency>
<groupId>com.bucket4j</groupId>
<artifactId>bucket4j_jdk17-core</artifactId>
</dependency>
</dependencies>
<build>
@@ -132,14 +136,14 @@
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<phase>process-sources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmTarget>1.8</jvmTarget>
<jvmTarget>21</jvmTarget>
<compilerPlugins>
<plugin>kotlinx-serialization</plugin>
</compilerPlugins>
@@ -1,11 +1,18 @@
package me.vripper
import me.vripper.download.DownloadService
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.data.repositories.impl.ImageRepositoryImpl
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
import me.vripper.data.repositories.impl.PostRepositoryImpl
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
import me.vripper.event.EventBus
import me.vripper.host.*
import me.vripper.repositories.*
import me.vripper.repositories.impl.*
import me.vripper.services.*
import me.vripper.services.download.DownloadService
import me.vripper.services.download.QueueManager
import org.koin.core.qualifier.named
import org.koin.dsl.bind
import org.koin.dsl.module
@@ -17,11 +24,14 @@ val coreModule = module {
single<SettingsService> {
SettingsService(get())
}
single<LogService> {
LogService(get())
}
single<ImageRepository> {
ImageRepositoryImpl()
}
single<PostDownloadStateRepository> {
PostDownloadStateRepositoryImpl()
single<PostRepository> {
PostRepositoryImpl()
}
single<MetadataRepository> {
MetadataRepositoryImpl()
@@ -29,88 +39,88 @@ val coreModule = module {
single<ThreadRepository> {
ThreadRepositoryImpl()
}
single<LogRepository> {
LogRepositoryImpl(get())
}
single<DataTransaction> {
DataTransaction(get(), get(), get(), get(), get(), get(), get())
single<DataAccessService> {
DataAccessService(get(), get(), get(), get(), get(), get())
}
single<RetryPolicyService> {
RetryPolicyService(get(), get())
RetryPolicyService()
}
single<HTTPService> {
HTTPService(get(), get())
HTTPService()
}
single<VGAuthService> {
VGAuthService(get(), get(), get())
VGAuthService(get(), get())
}
single<ThreadCacheService> {
ThreadCacheService(get())
}
single<QueueManager> {
QueueManager(get(), getAll(), get(), get(), get())
}
single<DownloadService> {
DownloadService(get(), get(), get(), get(), get())
DownloadService(get(), get(), get(), get())
}
single<DownloadSpeedService> {
DownloadSpeedService(get())
}
single<AppEndpointService> {
AppEndpointService(get(), get(), get(), get(), get())
AppEndpointService(get(), get(), get(), get(), get(), get(), get())
}
single((named("localAppEndpointService"))) {
AppEndpointService(get(), get(), get(), get(), get())
AppEndpointService(get(), get(), get(), get(), get(), get(), get())
} bind IAppEndpointService::class
single<MetadataService> {
MetadataService(get(), get(), get(), get())
MetadataService(get(), get())
}
single {
AcidimgHost(get(), get(), get())
AcidimgHost()
} bind Host::class
single {
DPicMeHost(get(), get(), get())
DPicMeHost()
} bind Host::class
single {
ImageBamHost(get(), get(), get())
ImageBamHost()
} bind Host::class
single {
ImageTwistHost(get(), get(), get())
ImageTwistHost()
} bind Host::class
single {
ImageVenueHost(get(), get(), get())
ImageVenueHost()
} bind Host::class
single {
ImageZillaHost(get(), get(), get())
ImageZillaHost()
} bind Host::class
single {
ImgboxHost(get(), get(), get())
ImgboxHost()
} bind Host::class
single {
ImgSpiceHost(get(), get(), get())
ImgSpiceHost()
} bind Host::class
single {
ImxHost(get(), get(), get())
ImxHost()
} bind Host::class
single {
PimpandhostHost(get(), get(), get())
PimpandhostHost()
} bind Host::class
single {
PixhostHost(get(), get(), get())
PixhostHost()
} bind Host::class
single {
PixRouteHost(get(), get(), get())
PixRouteHost()
} bind Host::class
single {
PixxxelsHost(get(), get(), get())
PixxxelsHost()
} bind Host::class
single {
PostImgHost(get(), get(), get())
PostImgHost()
} bind Host::class
single {
TurboImageHost(get(), get(), get())
TurboImageHost()
} bind Host::class
single {
ViprImHost(get(), get(), get())
ViprImHost()
} bind Host::class
}
@@ -0,0 +1,16 @@
package me.vripper
import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.AppenderBase
import org.apache.commons.collections4.queue.CircularFifoQueue
class RingAppender : AppenderBase<ILoggingEvent>() {
object Static {
val events = CircularFifoQueue<ILoggingEvent>()
}
override fun append(event: ILoggingEvent) {
Static.events.offer(event)
}
}
@@ -0,0 +1,20 @@
package me.vripper.data.repositories
import me.vripper.entities.ImageEntity
import java.util.*
internal interface ImageRepository {
fun save(imageEntity: ImageEntity): ImageEntity
fun save(imageEntityList: List<ImageEntity>)
fun deleteAllByPostEntityId(postEntityId: Long)
fun findByPostEntityId(postEntityId: Long): List<ImageEntity>
fun countError(): Int
fun findByPostEntityIdAndIsNotCompleted(postEntityId: Long): List<ImageEntity>
fun stopByPostEntityIdAndIsNotCompleted(postEntityId: Long): Int
fun stopByPostEntityIdAndIsNotCompleted(): Int
fun findByPostEntityIdAndIsError(postEntityId: Long): List<ImageEntity>
fun findById(id: Long): Optional<ImageEntity>
fun update(imageEntity: ImageEntity)
fun update(imageEntities: List<ImageEntity>)
fun deleteAllByPostEntityId(postEntityIds: List<Long>)
}
@@ -0,0 +1,11 @@
package me.vripper.data.repositories
import me.vripper.entities.MetadataEntity
import java.util.*
internal interface MetadataRepository {
fun save(metadataEntity: MetadataEntity): MetadataEntity
fun findByPostEntityId(postEntityId: Long): Optional<MetadataEntity>
fun deleteByPostEntityId(postEntityId: Long): Int
fun deleteAllByPostEntityId(postEntityIds: List<Long>)
}
@@ -1,21 +1,19 @@
package me.vripper.repositories
package me.vripper.data.repositories
import me.vripper.entities.PostEntity
import java.util.*
interface PostDownloadStateRepository {
internal interface PostRepository {
fun save(postEntities: List<PostEntity>): List<PostEntity>
fun findByPostId(postId: Long): Optional<PostEntity>
fun findById(id: Long): Optional<PostEntity>
fun findById(id: Long): PostEntity?
fun findCompleted(): List<Long>
fun findAll(): List<PostEntity>
fun existByPostEntityId(postEntityId: Long): Boolean
fun existByPostId(postId: Long): Boolean
fun setDownloadingToStopped(): Int
fun deleteByPostId(postId: Long): Int
fun deleteByPostEntityId(postEntityId: Long): Int
fun update(postEntity: PostEntity)
fun update(postEntities: List<PostEntity>)
fun findMaxRank(): Int?
fun deleteAll(postIds: List<Long>)
fun deleteAll(postEntityIds: List<Long>)
fun stopAll()
fun findAllNonCompletedPostIds(): List<Long>
fun findAllNonCompletedPostEntityIds(): List<Long>
}
@@ -1,9 +1,9 @@
package me.vripper.repositories
package me.vripper.data.repositories
import me.vripper.entities.ThreadEntity
import java.util.*
interface ThreadRepository {
internal interface ThreadRepository {
fun save(threadEntity: ThreadEntity): ThreadEntity
fun update(threadEntity: ThreadEntity)
fun findByThreadId(threadId: Long): Optional<ThreadEntity>
@@ -1,27 +1,26 @@
package me.vripper.repositories.impl
package me.vripper.data.repositories.impl
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.tables.ImageTable
import me.vripper.entities.ImageEntity
import me.vripper.entities.domain.Status
import me.vripper.repositories.ImageRepository
import me.vripper.tables.ImageTable
import me.vripper.entities.Status
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.transactions.TransactionManager
import java.sql.Connection
import java.util.*
class ImageRepositoryImpl : ImageRepository {
internal class ImageRepositoryImpl : ImageRepository {
override fun save(imageEntity: ImageEntity): ImageEntity {
val id = ImageTable.insertAndGetId {
it[downloaded] = imageEntity.downloaded
it[host] = imageEntity.host
it[index] = imageEntity.index
it[postId] = imageEntity.postId
it[status] = imageEntity.status.name
it[size] = imageEntity.size
it[url] = imageEntity.url
it[thumbUrl] = imageEntity.thumbUrl
it[postIdRef] = imageEntity.postIdRef
it[postIdRef] = imageEntity.postEntityId
it[filename] = imageEntity.filename
}.value
return imageEntity.copy(id = id)
@@ -32,60 +31,59 @@ class ImageRepositoryImpl : ImageRepository {
this[ImageTable.downloaded] = it.downloaded
this[ImageTable.host] = it.host
this[ImageTable.index] = it.index
this[ImageTable.postId] = it.postId
this[ImageTable.status] = it.status.name
this[ImageTable.size] = it.size
this[ImageTable.url] = it.url
this[ImageTable.thumbUrl] = it.thumbUrl
this[ImageTable.postIdRef] = it.postIdRef
this[ImageTable.postIdRef] = it.postEntityId
this[ImageTable.filename] = it.filename
}
}
override fun deleteAllByPostId(postId: Long) {
ImageTable.deleteWhere { ImageTable.postId eq postId }
override fun deleteAllByPostEntityId(postEntityId: Long) {
ImageTable.deleteWhere { ImageTable.postIdRef eq postEntityId }
}
override fun findByPostId(postId: Long): List<ImageEntity> {
return ImageTable.select {
ImageTable.postId eq postId
override fun findByPostEntityId(postEntityId: Long): List<ImageEntity> {
return ImageTable.selectAll().where {
ImageTable.postIdRef eq postEntityId
}.map(this::transform)
}
override fun countError(): Int {
return ImageTable
.slice(ImageTable.id)
.select { ImageTable.status eq Status.ERROR.name }
.select(ImageTable.id)
.where { ImageTable.status eq Status.ERROR.name }
.count().toInt()
}
override fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity> {
override fun findByPostEntityIdAndIsNotCompleted(postEntityId: Long): List<ImageEntity> {
return ImageTable
.select {
(ImageTable.postId eq postId) and (ImageTable.status neq Status.FINISHED.name)
.selectAll().where {
(ImageTable.postIdRef eq postEntityId) and (ImageTable.status neq Status.FINISHED.name)
}.map(this::transform)
}
override fun stopByPostIdAndIsNotCompleted(postId: Long): Int {
return ImageTable.update({ (ImageTable.postId eq postId) and (ImageTable.status neq Status.FINISHED.name) }) {
override fun stopByPostEntityIdAndIsNotCompleted(postEntityId: Long): Int {
return ImageTable.update({ (ImageTable.postIdRef eq postEntityId) and (ImageTable.status neq Status.FINISHED.name) }) {
it[status] = Status.STOPPED.name
}
}
override fun stopByPostIdAndIsNotCompleted(): Int {
override fun stopByPostEntityIdAndIsNotCompleted(): Int {
return ImageTable.update({ (ImageTable.status neq Status.FINISHED.name) }) {
it[status] = Status.STOPPED.name
}
}
override fun findByPostIdAndIsError(postId: Long): List<ImageEntity> {
return ImageTable.select {
(ImageTable.postId eq postId) and (ImageTable.status eq Status.ERROR.name)
override fun findByPostEntityIdAndIsError(postEntityId: Long): List<ImageEntity> {
return ImageTable.selectAll().where {
(ImageTable.postIdRef eq postEntityId) and (ImageTable.status eq Status.ERROR.name)
}.map(this::transform)
}
override fun findById(id: Long): Optional<ImageEntity> {
val result = ImageTable.select {
val result = ImageTable.selectAll().where {
ImageTable.id eq id
}.map(this::transform)
return if (result.isEmpty()) {
@@ -110,44 +108,42 @@ class ImageRepositoryImpl : ImageRepository {
this[ImageTable.downloaded] = it.downloaded
this[ImageTable.host] = it.host
this[ImageTable.index] = it.index
this[ImageTable.postId] = it.postId
this[ImageTable.status] = it.status.name
this[ImageTable.size] = it.size
this[ImageTable.url] = it.url
this[ImageTable.thumbUrl] = it.thumbUrl
this[ImageTable.postIdRef] = it.postIdRef
this[ImageTable.postIdRef] = it.postEntityId
this[ImageTable.filename] = it.filename
}
}
override fun deleteAllByPostId(postIds: List<Long>) {
override fun deleteAllByPostEntityId(postEntityIds: List<Long>) {
val conn = TransactionManager.current().connection.connection as Connection
conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE IMAGES_DELETE(POST_ID BIGINT PRIMARY KEY)")
conn.prepareStatement("CREATE TEMPORARY TABLE IMAGES_DELETE(POST_ID_REF BIGINT PRIMARY KEY)")
.use {
it.execute()
}
conn.prepareStatement("INSERT INTO IMAGES_DELETE VALUES ( ? )").use { ps ->
postIds.forEach {
postEntityIds.forEach {
ps.setLong(1, it)
ps.addBatch()
}
ps.executeBatch()
}
conn.prepareStatement("DELETE FROM IMAGE WHERE POST_ID IN (SELECT POST_ID FROM IMAGES_DELETE)")
conn.prepareStatement("DELETE FROM IMAGE WHERE POST_ID_REF IN (SELECT POST_ID_REF FROM IMAGES_DELETE)")
.use {
it.execute()
}
conn.prepareStatement("TRUNCATE TABLE IMAGES_DELETE").use {
conn.prepareStatement("DELETE FROM IMAGES_DELETE").use {
it.execute()
}
}
private fun transform(resultRow: ResultRow): ImageEntity {
val id = resultRow[ImageTable.id].value
val postId = resultRow[ImageTable.postId]
val url = resultRow[ImageTable.url]
val thumbUrl = resultRow[ImageTable.thumbUrl]
val host = resultRow[ImageTable.host]
@@ -159,7 +155,6 @@ class ImageRepositoryImpl : ImageRepository {
val filename = resultRow[ImageTable.filename]
return ImageEntity(
id,
postId,
url,
thumbUrl,
host,
@@ -1,34 +1,34 @@
package me.vripper.repositories.impl
package me.vripper.data.repositories.impl
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.tables.MetadataTable
import me.vripper.entities.MetadataEntity
import me.vripper.repositories.MetadataRepository
import me.vripper.tables.MetadataTable
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.TransactionManager
import java.sql.Connection
import java.util.*
class MetadataRepositoryImpl: MetadataRepository {
internal class MetadataRepositoryImpl : MetadataRepository {
override fun save(metadataEntity: MetadataEntity): MetadataEntity {
MetadataTable.insert {
it[postId] = metadataEntity.postId
it[postIdRef] = metadataEntity.postIdRef
it[data] = Json.encodeToString(metadataEntity.data)
}
return metadataEntity
}
override fun findByPostId(postId: Long): Optional<MetadataEntity> {
override fun findByPostEntityId(postEntityId: Long): Optional<MetadataEntity> {
val result = MetadataTable.select {
MetadataTable.postId eq postId
val result = MetadataTable.selectAll().where {
MetadataTable.postIdRef eq postEntityId
}.map(::transform)
return if (result.isEmpty()) {
@@ -39,36 +39,36 @@ class MetadataRepositoryImpl: MetadataRepository {
}
private fun transform(row: ResultRow): MetadataEntity {
val id = row[MetadataTable.postId]
val postEntityId = row[MetadataTable.postIdRef]
val data = Json.decodeFromString(row[MetadataTable.data]) as MetadataEntity.Data
return MetadataEntity(id, data)
return MetadataEntity(postEntityId, data)
}
override fun deleteByPostId(postId: Long): Int {
return MetadataTable.deleteWhere { MetadataTable.postId eq postId }
override fun deleteByPostEntityId(postEntityId: Long): Int {
return MetadataTable.deleteWhere { MetadataTable.postIdRef eq postEntityId }
}
override fun deleteAllByPostId(postIds: List<Long>) {
override fun deleteAllByPostEntityId(postEntityIds: List<Long>) {
val conn = TransactionManager.current().connection.connection as Connection
conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE METADATA_DELETE(POST_ID BIGINT PRIMARY KEY)")
conn.prepareStatement("CREATE TEMPORARY TABLE METADATA_DELETE(POST_ID_REF BIGINT PRIMARY KEY)")
.use {
it.execute()
}
conn.prepareStatement("INSERT INTO METADATA_DELETE VALUES ( ? )").use { ps ->
postIds.forEach {
postEntityIds.forEach {
ps.setLong(1, it)
ps.addBatch()
}
ps.executeBatch()
}
conn.prepareStatement("DELETE FROM METADATA WHERE POST_ID IN (SELECT POST_ID FROM METADATA_DELETE)")
conn.prepareStatement("DELETE FROM METADATA WHERE POST_ID_REF IN (SELECT POST_ID_REF FROM METADATA_DELETE)")
.use {
it.execute()
}
conn.prepareStatement("TRUNCATE TABLE METADATA_DELETE").use {
conn.prepareStatement("DELETE FROM METADATA_DELETE").use {
it.execute()
}
}
@@ -1,17 +1,16 @@
package me.vripper.repositories.impl
package me.vripper.data.repositories.impl
import me.vripper.data.repositories.PostRepository
import me.vripper.data.tables.PostTable
import me.vripper.entities.PostEntity
import me.vripper.entities.domain.Status
import me.vripper.repositories.PostDownloadStateRepository
import me.vripper.tables.PostTable
import me.vripper.entities.Status
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.transactions.TransactionManager
import java.sql.Connection
import java.util.*
class PostDownloadStateRepositoryImpl :
PostDownloadStateRepository {
internal class PostRepositoryImpl :
PostRepository {
private val delimiter = ";"
@@ -20,12 +19,11 @@ class PostDownloadStateRepositoryImpl :
this[PostTable.status] = post.status.name
this[PostTable.done] = post.done
this[PostTable.total] = post.total
this[PostTable.rank] = post.rank
this[PostTable.hosts] = post.hosts.joinToString(delimiter)
this[PostTable.outputPath] = post.downloadDirectory
this[PostTable.folderName] = post.folderName
this[PostTable.postId] = post.postId
this[PostTable.threadId] = post.threadId
this[PostTable.postId] = post.vgPostId
this[PostTable.threadId] = post.vgThreadId
this[PostTable.postTitle] = post.postTitle
this[PostTable.threadTitle] = post.threadTitle
this[PostTable.forum] = post.forum
@@ -37,41 +35,29 @@ class PostDownloadStateRepositoryImpl :
}.map(::transform)
}
override fun findByPostId(postId: Long): Optional<PostEntity> {
val result = PostTable.select {
PostTable.postId eq postId
}.map(::transform)
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
}
override fun findCompleted(): List<Long> {
return PostTable.slice(PostTable.postId).select {
return PostTable.select(PostTable.id).where {
(PostTable.status eq Status.FINISHED.name) and (PostTable.done greaterEq PostTable.total)
}.map { it[PostTable.postId] }
}.map { it[PostTable.id].value }
}
override fun findById(id: Long): Optional<PostEntity> {
val result = PostTable.select {
override fun findById(id: Long): PostEntity? {
val result = PostTable.selectAll().where {
PostTable.id eq id
}.map { transform(it) }
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
return result.firstOrNull()
}
override fun findAll(): List<PostEntity> {
return PostTable.selectAll().map { transform(it) }
}
override fun existByPostEntityId(postEntityId: Long): Boolean {
return PostTable.select(PostTable.id).where { PostTable.id eq postEntityId }.count() > 0
}
override fun existByPostId(postId: Long): Boolean {
return PostTable.slice(PostTable.id).select { PostTable.postId eq postId }.count() > 0
return PostTable.select(PostTable.postId).where { PostTable.postId eq postId }.count() > 0
}
override fun setDownloadingToStopped(): Int {
@@ -80,15 +66,14 @@ class PostDownloadStateRepositoryImpl :
}
}
override fun deleteByPostId(postId: Long): Int {
return PostTable.deleteWhere { PostTable.postId eq postId }
override fun deleteByPostEntityId(postEntityId: Long): Int {
return PostTable.deleteWhere { PostTable.id eq postEntityId }
}
override fun update(postEntity: PostEntity) {
PostTable.update({ PostTable.id eq postEntity.id }) {
it[status] = postEntity.status.name
it[done] = postEntity.done
it[rank] = postEntity.rank
it[size] = postEntity.size
it[downloaded] = postEntity.downloaded
it[folderName] = postEntity.folderName
@@ -101,12 +86,11 @@ class PostDownloadStateRepositoryImpl :
this[PostTable.status] = post.status.name
this[PostTable.done] = post.done
this[PostTable.total] = post.total
this[PostTable.rank] = post.rank
this[PostTable.hosts] = post.hosts.joinToString(delimiter)
this[PostTable.outputPath] = post.downloadDirectory
this[PostTable.folderName] = post.folderName
this[PostTable.postId] = post.postId
this[PostTable.threadId] = post.threadId
this[PostTable.postId] = post.vgPostId
this[PostTable.threadId] = post.vgThreadId
this[PostTable.postTitle] = post.postTitle
this[PostTable.threadTitle] = post.threadTitle
this[PostTable.forum] = post.forum
@@ -114,37 +98,31 @@ class PostDownloadStateRepositoryImpl :
this[PostTable.token] = post.token
this[PostTable.size] = post.size
this[PostTable.downloaded] = post.downloaded
this[PostTable.addedAt] = post.addedOn
}
}
override fun findMaxRank(): Int? {
return PostTable.slice(PostTable.rank.max())
.selectAll()
.firstOrNull()
?.get(PostTable.rank.max())
}
override fun deleteAll(postIds: List<Long>) {
override fun deleteAll(postEntityIds: List<Long>) {
val conn = TransactionManager.current().connection.connection as Connection
conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE POSTS_DELETE(POST_ID BIGINT PRIMARY KEY)")
conn.prepareStatement("CREATE TEMPORARY TABLE POSTS_DELETE(ID BIGINT PRIMARY KEY)")
.use {
it.execute()
}
conn.prepareStatement("INSERT INTO POSTS_DELETE VALUES ( ? )").use { ps ->
postIds.forEach {
postEntityIds.forEach {
ps.setLong(1, it)
ps.addBatch()
}
ps.executeBatch()
}
conn.prepareStatement("DELETE FROM POST WHERE POST_ID IN (SELECT POST_ID FROM POSTS_DELETE)")
conn.prepareStatement("DELETE FROM POST WHERE ID IN (SELECT ID FROM POSTS_DELETE)")
.use {
it.execute()
}
conn.prepareStatement("TRUNCATE TABLE POSTS_DELETE").use {
conn.prepareStatement("DELETE FROM POSTS_DELETE").use {
it.execute()
}
}
@@ -155,10 +133,10 @@ class PostDownloadStateRepositoryImpl :
}
}
override fun findAllNonCompletedPostIds(): List<Long> {
return PostTable.slice(PostTable.postId).select {
override fun findAllNonCompletedPostEntityIds(): List<Long> {
return PostTable.select(PostTable.id).where {
PostTable.status neq Status.FINISHED.name
}.map { it[PostTable.postId] }
}.map { it[PostTable.id].value }
}
private fun transform(resultRow: ResultRow): PostEntity {
@@ -178,7 +156,6 @@ class PostDownloadStateRepositoryImpl :
val downloadDirectory = resultRow[PostTable.outputPath]
val folderName = resultRow[PostTable.folderName]
val addedOn = resultRow[PostTable.addedAt]
val rank = resultRow[PostTable.rank]
val size = resultRow[PostTable.size]
val downloaded = resultRow[PostTable.downloaded]
return PostEntity(
@@ -197,7 +174,6 @@ class PostDownloadStateRepositoryImpl :
folderName,
status,
done,
rank,
size,
downloaded
)
@@ -1,13 +1,13 @@
package me.vripper.repositories.impl
package me.vripper.data.repositories.impl
import me.vripper.data.repositories.ThreadRepository
import me.vripper.data.tables.ThreadTable
import me.vripper.entities.ThreadEntity
import me.vripper.repositories.ThreadRepository
import me.vripper.tables.ThreadTable
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import java.util.*
class ThreadRepositoryImpl : ThreadRepository {
internal class ThreadRepositoryImpl : ThreadRepository {
override fun save(threadEntity: ThreadEntity): ThreadEntity {
val id = ThreadTable.insertAndGetId {
@@ -27,7 +27,7 @@ class ThreadRepositoryImpl : ThreadRepository {
}
override fun findByThreadId(threadId: Long): Optional<ThreadEntity> {
val result = ThreadTable.select {
val result = ThreadTable.selectAll().where {
ThreadTable.threadId eq threadId
}.map(this::transform)
return if (result.isEmpty()) {
@@ -42,7 +42,7 @@ class ThreadRepositoryImpl : ThreadRepository {
}
override fun findById(id: Long): Optional<ThreadEntity> {
val result = ThreadTable.select {
val result = ThreadTable.selectAll().where {
ThreadTable.id eq id
}.map(this::transform)
return if (result.isEmpty()) {
@@ -1,16 +1,15 @@
package me.vripper.tables
package me.vripper.data.tables
import org.jetbrains.exposed.dao.id.LongIdTable
object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") {
internal object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") {
val downloaded = long("DOWNLOADED")
val host = byte("HOST")
val index = integer("INDEX")
val postId = long("POST_ID")
val status = varchar("STATUS", 15)
val filename = varchar("FILENAME", 260)
val size = long("SIZE")
val url = varchar("URL", 200)
val thumbUrl = varchar("THUMB_URL", 200)
val postIdRef = long("POST_ID_REF").references(PostTable.id, fkName = "IMAGE_POST_ID_REF_POST_ID_FK")
val postIdRef = long("POST_ID_REF").references(id, fkName = "IMAGE_POST_ID_REF_POST_ID_FK")
}
@@ -0,0 +1,8 @@
package me.vripper.data.tables
import org.jetbrains.exposed.sql.Table
internal object MetadataTable : Table(name = "METADATA") {
val postIdRef = long("POST_ID_REF").references(PostTable.id, fkName = "METADATA_POST_ID_REF_POST_ID_FK")
val data = varchar("DATA", 1_000_000)
}
@@ -1,10 +1,10 @@
package me.vripper.tables
package me.vripper.data.tables
import org.jetbrains.exposed.dao.id.LongIdTable
import org.jetbrains.exposed.sql.javatime.datetime
import java.time.LocalDateTime
object PostTable : LongIdTable(name = "POST", columnName = "ID") {
internal object PostTable : LongIdTable(name = "POST", columnName = "ID") {
val done = integer("DONE")
val hosts = varchar("HOSTS", 255)
val outputPath = varchar("OUTPUT_PATH", 260)
@@ -1,8 +1,8 @@
package me.vripper.tables
package me.vripper.data.tables
import org.jetbrains.exposed.dao.id.LongIdTable
object ThreadTable : LongIdTable("THREAD", columnName = "ID") {
internal object ThreadTable : LongIdTable("THREAD", columnName = "ID") {
val total = integer("TOTAL").default(0)
val url = varchar("URL", 200)
val threadId = long("THREAD_ID")
@@ -1,299 +0,0 @@
package me.vripper.download
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Runnable
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.entities.ImageEntity
import me.vripper.entities.LogEntryEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.domain.Status
import me.vripper.event.ErrorCountEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.event.StoppedEvent
import me.vripper.host.Host
import me.vripper.model.ErrorCount
import me.vripper.model.QueueState
import me.vripper.services.DataTransaction
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.GLOBAL_EXECUTOR
import me.vripper.utilities.formatToString
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.RetryPolicy
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.concurrent.CompletableFuture
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
class DownloadService(
private val settingsService: SettingsService,
private val dataTransaction: DataTransaction,
private val retryPolicyService: RetryPolicyService,
private val vgAuthService: VGAuthService,
private val eventBus: EventBus
) {
private val maxPoolSize: Int = 24
private val log by me.vripper.delegate.LoggerDelegate()
private val running: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
private val lock = ReentrantLock()
private val condition = lock.newCondition()
private val coroutineScope = CoroutineScope(SupervisorJob())
init {
Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
while (!Thread.currentThread().isInterrupted) {
lock.withLock {
candidates.addAll(getCandidates(candidateCount()))
candidates.forEach {
if (canRun(it.context.imageEntity.host)) {
accepted.add(it)
running[it.context.imageEntity.host]!!.add(it)
log.debug("${it.context.imageEntity.url} accepted to run")
}
}
accepted.forEach {
pending[it.context.imageEntity.host]?.remove(it)
scheduleForDownload(it)
}
accepted.clear()
candidates.clear()
try {
condition.await()
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
}).start()
}
fun stop(postIds: List<Long> = emptyList()) {
if (postIds.isNotEmpty()) {
stopInternal(postIds)
coroutineScope.launch {
eventBus.publishEvent(StoppedEvent(postIds))
}
} else {
stopAll()
coroutineScope.launch {
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
}
}
fun restartAll(postEntityIds: List<PostEntity> = emptyList()) {
if (postEntityIds.isNotEmpty()) {
restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
} else {
restart(dataTransaction.findAllPosts()
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
}
}
private fun restart(posts: Map<PostEntity, List<ImageEntity>>) {
lock.withLock {
val toProcess = mutableMapOf<PostEntity, List<ImageEntity>>()
for ((post, images) in posts) {
if (images.isEmpty()) {
continue
}
if (isPending(post.postId)) {
continue
}
toProcess[post] = images
}
toProcess.forEach { (post, images) ->
post.status = Status.PENDING
images.forEach { image ->
with(image) {
this.status = Status.PENDING
this.downloaded = 0
}
}
}
transaction {
dataTransaction.updatePosts(
toProcess.keys.toList()
)
dataTransaction.updateImages(
toProcess.values.flatten()
)
}
toProcess.keys.forEach {
vgAuthService.leaveThanks(it)
}
toProcess.entries.forEach { (post, images) ->
images.forEach { image ->
log.debug("Enqueuing a job for ${image.url}")
val imageDownloadRunnable = ImageDownloadRunnable(
image, post.rank, settingsService.settings
)
pending.computeIfAbsent(
image.host
) { mutableListOf() }
pending[image.host]!!.add(imageDownloadRunnable)
}
}
condition.signal()
}
}
private fun isPending(postId: Long): Boolean {
return pending.values.flatten().any { it.context.imageEntity.postId == postId }
}
private fun isRunning(postId: Long): Boolean {
return running.values.flatten().any { it.context.imageEntity.postId == postId }
}
private fun stopAll() {
lock.withLock {
pending.values.clear()
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
dataTransaction.findAllNonCompletedPostIds().forEach {
dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
dataTransaction.finishPost(it)
}
}
}
private fun stopInternal(postIds: List<Long>) {
lock.withLock {
for (postId in postIds) {
pending.values.forEach { pending ->
pending.removeIf { it.context.imageEntity.postId == postId }
}
running.values.flatten()
.filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
}
postIds.forEach {
dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
dataTransaction.finishPost(it)
}
}
}
private fun canRun(host: Byte): Boolean {
val totalRunning = running.values.sumOf { it.size }
return (running[host]!!.size < settingsService.settings.connectionSettings.maxConcurrentPerHost && if (settingsService.settings.connectionSettings.maxGlobalConcurrent == 0) totalRunning < maxPoolSize else totalRunning < settingsService.settings.connectionSettings.maxGlobalConcurrent)
}
private fun candidateCount(): Map<Byte, Int> {
val map: MutableMap<Byte, Int> = mutableMapOf()
Host.getHosts().values.forEach { host: Byte ->
val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
host
) { mutableListOf() }
val count: Int =
settingsService.settings.connectionSettings.maxConcurrentPerHost - imageDownloadRunnableList.size
log.debug("Download slots for $host: $count")
map[host] = count
}
return map
}
private fun getCandidates(candidateCount: Map<Byte, Int>): List<ImageDownloadRunnable> {
val hostIntegerMap: MutableMap<Byte, Int> = candidateCount.toMutableMap()
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
hosts@ for (host in pending.keys) {
val list: List<ImageDownloadRunnable> =
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
.thenComparingInt { it.context.imageEntity.index })
for (imageDownloadRunnable in list) {
val count = hostIntegerMap[host] ?: 0
if (count > 0) {
candidates.add(imageDownloadRunnable)
hostIntegerMap[host] = count - 1
} else {
continue@hosts
}
}
}
return candidates
}
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
CompletableFuture.runAsync({
coroutineScope.launch {
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
}
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload())
.onFailure {
try {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.DOWNLOAD,
status = LogEntryEntity.Status.ERROR,
message = "Failed to download ${imageDownloadRunnable.context.imageEntity.url}\n ${it.failure.formatToString()}"
)
)
} catch (exp: Exception) {
log.error("Failed to save event", exp)
}
log.error(
"Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
it.failure
)
val image = imageDownloadRunnable.context.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}.onComplete {
afterJobFinish(imageDownloadRunnable)
coroutineScope.launch {
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
)
)
}
coroutineScope.launch {
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
}
log.debug(
"Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
)
}.run(imageDownloadRunnable::run)
}, GLOBAL_EXECUTOR)
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
lock.withLock {
val image = imageDownloadRunnable.context.imageEntity
running[image.host]!!.remove(imageDownloadRunnable)
if (!isPending(image.postId) && !isRunning(
image.postId
) && !imageDownloadRunnable.context.stopped
) {
dataTransaction.finishPost(image.postId, true)
}
condition.signal()
}
}
private fun pendingCount(): Int {
return pending.values.sumOf { it.size }
}
private fun runningCount(): Int {
return running.values.sumOf { it.size }
}
}
@@ -1,17 +0,0 @@
package me.vripper.download
import me.vripper.entities.ImageEntity
import me.vripper.model.Settings
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.koin.core.component.KoinComponent
class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
val httpContext: HttpClientContext =
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
var stopped = false
}
@@ -1,149 +0,0 @@
package me.vripper.download
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.entities.domain.Status
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.*
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import net.jodah.failsafe.function.CheckedRunnable
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.*
import kotlin.io.path.Path
import kotlin.io.path.pathString
class ImageDownloadRunnable(
private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by me.vripper.delegate.LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val hosts: List<Host> = getKoin().getAll()
val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
private var stopped: Boolean
get() = context.stopped
set(value) {
context.stopped = value
}
@Throws(DownloadException::class)
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
}
}
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
val host = hosts.first { it.isSupported(imageEntity.url) }
val downloadedImage = host.downloadInternal(imageEntity.url, context)
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
)
if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
imageEntity.status = Status.FINISHED
post.done += 1
post.downloaded += imageEntity.size
dataTransaction.updatePost(post)
} else {
imageEntity.status = Status.ERROR
}
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (stopped) {
return
}
imageEntity.status = Status.ERROR
dataTransaction.updateImage(imageEntity)
throw DownloadException(e)
}
}
@Throws(HostException::class)
private fun checkImageTypeAndRename(
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
) {
val existingExtension = getExtension(downloadedImage.name).lowercase()
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
val extension = when (downloadedImage.type) {
ImageMimeType.IMAGE_BMP -> "BMP"
ImageMimeType.IMAGE_GIF -> "GIF"
ImageMimeType.IMAGE_JPEG -> "JPG"
ImageMimeType.IMAGE_PNG -> "PNG"
ImageMimeType.IMAGE_WEBP -> "WEBP"
}
val filename =
if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
sanitize(
fileNameWithoutExtension
)
}.$extension"
try {
val downloadDestinationFolder = Path.of(downloadDirectory)
Files.createDirectories(downloadDestinationFolder)
val finalFilename = "${
if (settings.downloadSettings.forceOrder) String.format(
"%03d_", index + 1
) else ""
}$filename"
imageEntity.filename = finalFilename
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
} catch (e: Exception) {
throw HostException("Failed to rename the image", e)
} finally {
try {
Files.delete(downloadedImage.path)
} catch (_: IOException) {
}
}
}
@Throws(Exception::class)
override fun run() {
if (stopped) {
return
}
download()
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
val that = other as ImageDownloadRunnable
return imageEntity.id == that.imageEntity.id
}
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
fun stop() {
context.requests.forEach { it.abort() }
stopped = true
}
}
@@ -1,32 +1,17 @@
package me.vripper.entities
import me.vripper.entities.domain.Status
import kotlinx.serialization.Serializable
@Serializable
data class ImageEntity(
val id: Long = -1,
val postId: Long,
val url: String,
val thumbUrl: String,
val host: Byte,
val index: Int,
val postIdRef: Long = -1,
val postEntityId: Long = -1,
var size: Long = -1,
var downloaded: Long = 0,
var status: Status = Status.STOPPED,
var filename: String = "",
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as ImageEntity
if (url != other.url) return false
return true
}
override fun hashCode(): Int {
return url.hashCode()
}
}
)
@@ -1,22 +1,14 @@
package me.vripper.entities
import java.time.LocalDateTime
import kotlinx.serialization.Serializable
@Serializable
data class LogEntryEntity(
val id: Long = -1,
val type: Type,
val status: Status,
val time: LocalDateTime = LocalDateTime.now(),
val message: String,
) {
enum class Type(val stringValue: String) {
POST("Post"), THREAD("Thread"), THANKS("Thanks"), METADATA("Metadata"), SCAN("Scan"), DOWNLOAD(
"Download"
)
}
enum class Status(val stringValue: String) {
PENDING("Pending"), PROCESSING("Processing"), DONE("Done"), ERROR("Error")
}
}
val sequence: Long,
val timestamp: String,
val threadName: String,
val loggerName: String,
val levelString: String,
val formattedMessage: String,
val throwable: String,
)
@@ -2,7 +2,8 @@ package me.vripper.entities
import kotlinx.serialization.Serializable
data class MetadataEntity(val postId: Long, val data: Data) {
@Serializable
data class MetadataEntity(val postIdRef: Long, val data: Data) {
@Serializable
data class Data(
val postedBy: String,
@@ -1,8 +1,10 @@
package me.vripper.entities
import me.vripper.entities.domain.Status
import kotlinx.serialization.Contextual
import kotlinx.serialization.Serializable
import java.time.LocalDateTime
@Serializable
data class PostEntity(
val id: Long = -1,
val postTitle: String,
@@ -10,30 +12,15 @@ data class PostEntity(
val forum: String,
val url: String,
val token: String,
val postId: Long,
val threadId: Long,
val vgPostId: Long,
val vgThreadId: Long,
val total: Int,
val hosts: Set<String>,
val downloadDirectory: String,
val addedOn: LocalDateTime = LocalDateTime.now(),
@Contextual val addedOn: LocalDateTime = LocalDateTime.now(),
var folderName: String,
var status: Status = Status.STOPPED,
var done: Int = 0,
var rank: Int = Int.MAX_VALUE,
var size: Long = -1,
var downloaded: Long = 0,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as PostEntity
return postId == other.postId
}
override fun hashCode(): Int {
return postId.hashCode()
}
}
)
@@ -1,4 +1,4 @@
package me.vripper.entities.domain
package me.vripper.entities
enum class Status(val stringValue: String) {
PENDING("Pending"), DOWNLOADING("Downloading"), FINISHED("Finished"), ERROR("Error"), STOPPED("Stopped")
@@ -1,22 +1,12 @@
package me.vripper.entities
import kotlinx.serialization.Serializable
@Serializable
data class ThreadEntity(
val id: Long = -1,
val title: String,
val link: String,
val threadId: Long,
var total: Int = 0,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as ThreadEntity
return threadId == other.threadId
}
override fun hashCode(): Int {
return threadId.hashCode()
}
}
)
@@ -8,7 +8,7 @@ import me.vripper.model.Settings
data class PostCreateEvent(val postEntities: List<PostEntity>)
data class PostUpdateEvent(val postEntities: List<PostEntity>)
data class PostDeleteEvent(val postIds: List<Long>)
data class PostDeleteEvent(val postEntityIds: List<Long>)
data class ImageEvent(val imageEntities: List<ImageEntity>)
data class ThreadCreateEvent(val threadEntity: ThreadEntity)
data class ThreadUpdateEvent(val threadEntity: ThreadEntity)
@@ -21,7 +21,5 @@ data class ErrorCountEvent(val errorCount: ErrorCount)
data class SettingsUpdateEvent(val settings: Settings)
data class MetadataUpdateEvent(val metadataEntity: MetadataEntity)
data class LogCreateEvent(val logEntryEntity: LogEntryEntity)
data class LogUpdateEvent(val logEntryEntity: LogEntryEntity)
data class LogDeleteEvent(val deleted: List<Long>)
data class LoadingTasks(val loading: Boolean)
data class StoppedEvent(val postIds: List<Long>)
@@ -1,14 +1,15 @@
package me.vripper.event
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.asSharedFlow
object EventBus {
private val _events = MutableSharedFlow<Any>()
private val _events = MutableSharedFlow<Any>(0, Int.MAX_VALUE, BufferOverflow.DROP_OLDEST)
val events = _events.asSharedFlow()
suspend fun publishEvent(event: Any) {
_events.emit(event)
fun publishEvent(event: Any) {
_events.tryEmit(event)
}
}
@@ -1,41 +1,37 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable.Context
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.core5.http.message.BasicNameValuePair
import org.w3c.dom.Document
import org.w3c.dom.Node
class AcidimgHost(
private val httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("acidimg.cc", 0, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class AcidimgHost : Host("acidimg.cc", 0) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String, document: Document, context: ImageDownloadContext
context: Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
try {
log.debug(
String.format(
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, context.imageEntity.url
)
)
XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH)
} catch (e: XpathException) {
throw HostException(e)
}
log.debug(String.format("Click button found for %s", url))
val httpPost = HttpPost(url).also {
it.addHeader("Referer", url)
log.debug(String.format("Click button found for %s", context.imageEntity.url))
val httpPost = HttpPost(context.imageEntity.url).also {
it.addHeader("Referer", context.imageEntity.url)
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair(
@@ -44,8 +40,9 @@ class AcidimgHost(
)
)
)
it.setAbsoluteRequestUri(true)
}.also { context.requests.add(it) }
log.debug(String.format("Requesting %s", httpPost))
log.info("{}", httpPost)
val doc = try {
httpService.client.execute(
httpPost, context.httpContext
@@ -58,17 +55,17 @@ class AcidimgHost(
throw HostException(e)
}
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'", IMG_XPATH, url
"Xpath '%s' cannot be found in '%s'", IMG_XPATH, context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim()
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim()
Pair(
@@ -1,29 +1,23 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.util.*
class DPicMeHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("dpic.me", 1, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class DPicMeHost : Host("dpic.me", 1) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -31,11 +25,11 @@ class DPicMeHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle =
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { e: Node -> e.textContent.trim() }
@@ -1,28 +1,39 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.runBlocking
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.services.*
import me.vripper.services.DataAccessService
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.services.download.ImageDownloadRunnable.Context
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.client5.http.classic.methods.HttpHead
import org.apache.hc.core5.http.ClassicHttpResponse
import org.apache.hc.core5.http.Header
import org.jetbrains.exposed.sql.transactions.transaction
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.w3c.dom.Document
import java.io.BufferedOutputStream
import java.nio.file.Files
import java.nio.file.Path
abstract class Host(
internal abstract class Host(
val hostName: String,
val hostId: Byte,
private val httpService: HTTPService,
private val dataTransaction: DataTransaction,
private val downloadSpeedService: DownloadSpeedService
) {
private val log by me.vripper.delegate.LoggerDelegate()
) : KoinComponent {
private val log by LoggerDelegate()
protected val httpService: HTTPService by inject()
private val dataAccessService: DataAccessService by inject()
private val downloadSpeedService: DownloadSpeedService by inject()
companion object {
private const val READ_BUFFER_SIZE = 8192
@@ -37,54 +48,51 @@ abstract class Host(
hosts[hostName] = hostId
}
@Throws(HostException::class)
abstract fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: Context
): Pair<String, String>
@Throws(HostException::class)
fun downloadInternal(url: String, context: ImageDownloadContext): DownloadedImage {
val headers = head(url, context)
fun downloadInternal(context: Context): DownloadedImage {
if (hostId == 8.toByte()) {
return downloadByHost(context)
}
val headers = head(context)
// is the body of type image ?
val imageMimeType = getImageMimeType(headers)
val downloadedImage = if (imageMimeType != null) {
// a direct link, awesome
val downloadedImage = fetch(url, context) {
val downloadedImage = fetch(context.imageEntity.url, context) {
handleImageDownload(it, context)
}
DownloadedImage(getDefaultImageName(url), downloadedImage.first, downloadedImage.second)
DownloadedImage(getDefaultImageName(context.imageEntity.url), downloadedImage.first, downloadedImage.second)
} else {
// linked image ?
val value = headers.find { it.name.contains("content-type", true) }?.value
if (value != null) {
if (value.contains("text/html")) {
val document = fetch(url, context) {
HtmlUtils.clean(it.entity.content)
}
if (log.isDebugEnabled) {
log.debug("Cleaning $url response", url)
}
val resolvedImage = resolve(url, document, context)
val downloadImage: Pair<Path, ImageMimeType> =
fetch(resolvedImage.second, context) {
handleImageDownload(it, context)
}
DownloadedImage(resolvedImage.first, downloadImage.first, downloadImage.second)
downloadByHost(context)
} else {
throw HostException("Unable to download $url, can't process content type $value")
throw HostException("Unable to download ${context.imageEntity.url}, can't process content type $value")
}
} else {
throw HostException("Unexpected server response for $url, response have no content type")
throw HostException("Unexpected server response for ${context.imageEntity.url}, response have no content type")
}
}
return downloadedImage
}
private fun downloadByHost(context: Context): DownloadedImage {
val resolvedImage = resolve(context)
val downloadImage: Pair<Path, ImageMimeType> =
fetch(resolvedImage.second, context) {
handleImageDownload(it, context)
}
return DownloadedImage(resolvedImage.first, downloadImage.first, downloadImage.second)
}
private fun handleImageDownload(
response: ClassicHttpResponse,
context: ImageDownloadContext
context: Context
): Pair<Path, ImageMimeType> {
val mimeType = getImageMimeType(response.headers)
?: throw HostException("Unsupported image type ${response.getFirstHeader("content-type")}")
@@ -94,38 +102,46 @@ abstract class Host(
"vripper_",
".tmp"
)
return Files.newOutputStream(tempImage).use { fos ->
val image = context.imageEntity
synchronized(image.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val size = if (image.size < 0) {
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
synchronized(context.imageEntity.postEntityId.toString().intern()) {
val post = dataAccessService.findPostByEntityId(context.imageEntity.postEntityId)
val size = if (context.imageEntity.size < 0) {
response.entity.contentLength
} else {
0
}
image.size = response.entity.contentLength
context.imageEntity.size = response.entity.contentLength
post.size += size
transaction {
dataTransaction.updateImage(image)
dataTransaction.updatePost(post)
dataAccessService.updateImage(context.imageEntity)
dataAccessService.updatePost(post)
}
}
log.debug(
"Length is ${image.size}"
"Length is ${context.imageEntity.size}"
)
log.debug(
"Starting data transfer"
)
val buffer = ByteArray(READ_BUFFER_SIZE)
var read: Int
while (response.entity.content.read(buffer, 0, READ_BUFFER_SIZE)
.also { read = it } != -1 && !context.stopped
val reporterJob = context.launchCoroutine {
while (isActive) {
dataAccessService.updateImage(context.imageEntity, false)
delay(100)
}
}
while (response.entity.content.read(buffer)
.also { read = it } != -1
) {
fos.write(buffer, 0, read)
image.downloaded += read
dataTransaction.updateImage(image)
bos.write(buffer, 0, read)
context.imageEntity.downloaded += read
downloadSpeedService.reportDownloadedBytes(read.toLong())
}
runBlocking {
reporterJob.cancelAndJoin()
}
dataAccessService.updateImage(context.imageEntity)
Pair(tempImage, mimeType)
}
}
@@ -134,10 +150,13 @@ abstract class Host(
return url.contains(hostName)
}
@Throws(HostException::class)
fun head(url: String, context: ImageDownloadContext): Array<Header> {
val httpHead = HttpHead(url).also { context.requests.add(it) }
log.debug(String.format("Requesting %s", url))
fun head(context: Context): Array<Header> {
val httpHead = HttpHead(context.imageEntity.url).also {
it.addHeader("Referer", "https://vipergirls.to/")
it.setAbsoluteRequestUri(true)
context.requests.add(it)
}
log.info("{}", httpHead)
return httpService.client.execute(
httpHead,
context.httpContext
@@ -149,15 +168,18 @@ abstract class Host(
}
}
@Throws(HostException::class)
fun <T> fetch(
url: String,
context: ImageDownloadContext,
context: Context,
referer: String = "https://vipergirls.to/",
transformer: (ClassicHttpResponse) -> T
): T {
val httpGet =
HttpGet(url).also { it.addHeader("Referer", context.imageEntity.url) }.also { context.requests.add(it) }
log.debug(String.format("Requesting %s", url))
HttpGet(url).also {
it.addHeader("Referer", referer)
it.setAbsoluteRequestUri(true)
}.also { context.requests.add(it) }
log.info("{}", httpGet)
return httpService.client.execute(httpGet, context.httpContext) {
if (it.code / 100 != 2) {
throw DownloadException("Server returned code ${it.code}")
@@ -166,6 +188,19 @@ abstract class Host(
}
}
fun fetchDocument(
url: String,
context: Context
): Document {
return fetch(url, context) {
HtmlUtils.clean(it.entity.content)
}.also {
if (log.isDebugEnabled) {
log.debug("Cleaning {} response", url)
}
}
}
@OptIn(ExperimentalStdlibApi::class)
private fun getImageMimeType(headers: Array<Header>): ImageMimeType? {
@@ -1,44 +1,35 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.sql.Date
import java.time.LocalDateTime
import java.time.ZoneId
import java.util.*
class ImageBamHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imagebam.com", 2, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImageBamHost : Host("imagebam.com", 2) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val doc = try {
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, context.imageEntity.url))
if (XpathUtils.getAsNode(document, CONTINUE_XPATH) != null) {
val clientCookie = BasicClientCookie("nsfw_inter", "1")
clientCookie.domain = "www.imagebam.com"
clientCookie.path = "/"
clientCookie.expiryDate =
Date.from(
LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant()
)
context.httpContext.cookieStore.addCookie(clientCookie)
fetch(url, context) {
val sfwCookie = BasicClientCookie("sfw_inter", "1")
sfwCookie.domain = "www.imagebam.com"
sfwCookie.path = "/"
sfwCookie.setExpiryDate(LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant())
context.httpContext.cookieStore.addCookie(sfwCookie)
fetch(context.imageEntity.url, context) {
HtmlUtils.clean(it.entity.content)
}
} else {
@@ -48,7 +39,7 @@ class ImageBamHost(
throw HostException(e)
}
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -56,11 +47,11 @@ class ImageBamHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { e: Node -> e.textContent.trim { it <= ' ' } }
.orElse("")
@@ -1,29 +1,23 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.util.*
class ImageTwistHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imagetwist.com", 3, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImageTwistHost : Host("imagetwist.com", 3) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -31,11 +25,11 @@ class ImageTwistHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle =
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { obj: Node -> obj.textContent }
@@ -1,38 +1,32 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
class ImageVenueHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imagevenue.com", 4, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImageVenueHost : Host("imagevenue.com", 4) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val doc = try {
val document = fetchDocument(context.imageEntity.url, context)
log.debug(
String.format(
"Looking for xpath expression %s in %s",
CONTINUE_BUTTON_XPATH,
url
context.imageEntity.url
)
)
if (XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH) != null) {
// Button detected. No need to actually click it, just make the call again.
fetch(url, context) {
fetch(context.imageEntity.url, context) {
HtmlUtils.clean(it.entity.content)
}
} else {
@@ -42,7 +36,7 @@ class ImageVenueHost(
throw HostException(e)
}
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -50,11 +44,11 @@ class ImageVenueHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(
@@ -1,27 +1,21 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
class ImageZillaHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imagezilla.net", 5, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImageZillaHost : Host("imagezilla.net", 5) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val titleNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -29,17 +23,17 @@ class ImageZillaHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
log.debug(String.format("Resolving name for %s", url))
log.debug(String.format("Resolving name for %s", context.imageEntity.url))
var title = titleNode.attributes.getNamedItem("title").textContent.trim()
titleNode.textContent.trim()
if (title.isEmpty()) {
title = getDefaultImageName(url)
title = getDefaultImageName(context.imageEntity.url)
}
return try {
Pair(title, url.replace("show", "images"))
Pair(title, context.imageEntity.url.replace("show", "images"))
} catch (e: Exception) {
throw HostException("Unexpected error occurred", e)
}
@@ -1,28 +1,22 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
class ImgSpiceHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imgspice.com", 7, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImgSpiceHost : Host("imgspice.com", 7) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -30,11 +24,11 @@ class ImgSpiceHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(
@@ -1,28 +1,22 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
class ImgboxHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imgbox.com", 6, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImgboxHost : Host("imgbox.com", 6) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -30,11 +24,11 @@ class ImgboxHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("title").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(imgTitle, imgUrl)
@@ -1,35 +1,39 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.entities.ImageEntity
import me.vripper.exception.HostException
import me.vripper.exception.HtmlProcessorException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.core5.http.message.BasicNameValuePair
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.io.IOException
class ImxHost(
private val httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("imx.to", 8, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ImxHost : Host("imx.to", 8) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val httpsUrl = url.replace("http:", "https:")
var value: String? = null
try {
log.debug("Resolving name and image url for ${context.imageEntity.url}")
val imgTitle = getTitle(context).ifEmpty {
getDefaultImageName(context.imageEntity.thumbUrl)
}
val imgUrl = findPattern(context.imageEntity)
return Pair(
imgTitle, imgUrl
)
}
private fun getTitle(context: ImageDownloadRunnable.Context): String {
return try {
val httpsUrl = context.imageEntity.url.replace("http:", "https:")
val document = fetchDocument(httpsUrl, context)
var value: String? = null
log.debug("Looking for xpath expression $CONTINUE_BUTTON_XPATH in $httpsUrl")
val contDiv = XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH)
?: throw HostException("$CONTINUE_BUTTON_XPATH cannot be found")
@@ -37,46 +41,40 @@ class ImxHost(
if (node != null) {
value = node.textContent
}
} catch (e: XpathException) {
throw HostException(e)
}
if (value == null) {
throw HostException("Failed to obtain value attribute from continue input")
}
log.debug("Click button found for $httpsUrl")
val httpPost: HttpPost = HttpPost(httpsUrl).also {
it.entity = UrlEncodedFormEntity(listOf(BasicNameValuePair("imgContinue", value)))
}.also { context.requests.add(it) }
log.debug("Requesting {}", httpPost)
val doc = try {
httpService.client.execute(
log.debug("Click button found for $httpsUrl")
val httpPost: HttpPost = HttpPost(httpsUrl).also {
it.entity = UrlEncodedFormEntity(listOf(BasicNameValuePair("imgContinue", value)))
}.also { context.requests.add(it) }
log.debug("Requesting {}", httpPost)
val doc = httpService.client.execute(
httpPost, context.httpContext
) { response ->
log.debug("Cleaning response for {}", httpPost)
HtmlUtils.clean(response.entity.content)
}
} catch (e: IOException) {
throw HostException(e)
} catch (e: HtmlProcessorException) {
throw HostException(e)
}
val imgNode: Node = try {
log.debug("Looking for xpath expression $IMG_XPATH in $httpsUrl")
XpathUtils.getAsNode(doc, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
"Xpath $IMG_XPATH cannot be found in $httpsUrl"
)
return try {
log.debug("Resolving name and image url for $httpsUrl")
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
)
} catch (e: Exception) {
throw HostException("Unexpected error occurred", e)
val imgNode = XpathUtils.getAsNode(doc, IMG_XPATH)
log.debug("Resolving name for $httpsUrl")
val imgTitle = imgNode?.attributes?.getNamedItem("alt")?.textContent?.trim() ?: ""
return imgTitle
} catch (_: Exception) {
""
}
}
private fun findPattern(image: ImageEntity): String {
val url = image.thumbUrl
.replace("http:", "https:")
return if (url.startsWith("https://image.imx.to/u/t/")) {
"https://image.imx.to/u/i/" + url.replace("https://image.imx.to/u/t/", "")
} else if (url.startsWith("https://t.imx.to/t/")) {
"https://image.imx.to/u/i/" + url.replace("https://t.imx.to/t/", "")
} else if (url.startsWith("https://imx.to/upload/small/")) {
"https://image.imx.to/u/i/" + url.replace("https://imx.to/upload/small/", "")
} else {
throw HostException("Cannot find pattern for url ${image.thumbUrl}")
}
}
@@ -1,32 +1,28 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.net.URI
import java.net.URISyntaxException
class PimpandhostHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("pimpandhost.com", 9, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class PimpandhostHost : Host("pimpandhost.com", 9) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val newUrl: String
try {
newUrl = appendUri(url.replace("http://", "https://").replace("-medium(\\.html)?".toRegex(), ""), "size=original")
newUrl = appendUri(
context.imageEntity.url.replace("http://", "https://").replace("-medium(\\.html)?".toRegex(), ""),
"size=original"
)
} catch (e: Exception) {
throw HostException(e)
}
@@ -42,7 +38,7 @@ class PimpandhostHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
@@ -1,28 +1,22 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
class PixRouteHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("pixroute.com", 11, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class PixRouteHost : Host("pixroute.com", 11) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -30,11 +24,11 @@ class PixRouteHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
Pair(imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' },
imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
} catch (e: Exception) {
@@ -1,28 +1,22 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
class PixhostHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("pixhost.to", 10, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class PixhostHost : Host("pixhost.to", 10) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -30,11 +24,11 @@ class PixhostHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
Pair(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl)
@@ -1,27 +1,21 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
class PixxxelsHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("pixxxels.cc", 12, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class PixxxelsHost : Host("pixxxels.cc", 12) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -29,11 +23,11 @@ class PixxxelsHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
val titleNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, TITLE_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -41,11 +35,11 @@ class PixxxelsHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
TITLE_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle = titleNode.textContent.trim { it <= ' ' }
val imgUrl = imgNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' }
Pair(
@@ -1,64 +1,33 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.util.*
class PostImgHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("postimg.cc", 13, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class PostImgHost : Host("postimg.cc", 13) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val titleNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
XpathUtils.getAsNode(document, TITLE_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'",
TITLE_XPATH,
url
)
)
val urlNode = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
} ?: throw HostException(
val document = fetchDocument(context.imageEntity.url.replace("http:", "https:"), context)
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
val node = XpathUtils.getAsNode(document, IMG_XPATH) ?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
val imgTitle = Optional.ofNullable(titleNode)
.map { node: Node -> node.textContent.trim { it <= ' ' } }
.orElseGet { getDefaultImageName(url) }
Pair(imgTitle, urlNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' })
} catch (e: Exception) {
throw HostException("Unexpected error occurred", e)
}
return Pair(
node.attributes.getNamedItem("alt").textContent.trim(),
node.attributes.getNamedItem("src").textContent.trim()
)
}
companion object {
private const val TITLE_XPATH = "//span[contains(@class,'imagename')]"
private const val IMG_XPATH = "//a[@id='download']"
private const val IMG_XPATH = "//img[contains(@class,'img-fluid')]"
}
}
@@ -1,44 +1,38 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
class TurboImageHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("turboimagehost.com", 14, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class TurboImageHost : Host("turboimagehost.com", 14) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
var title: String?
title = try {
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, context.imageEntity.url))
val titleNode: Node? = XpathUtils.getAsNode(document, TITLE_XPATH)
log.debug(String.format("Resolving name for %s", url))
log.debug(String.format("Resolving name for %s", context.imageEntity.url))
titleNode?.textContent?.trim { it <= ' ' }
} catch (e: XpathException) {
throw HostException(e)
}
if (title.isNullOrEmpty()) {
title = getDefaultImageName(url)
title = getDefaultImageName(context.imageEntity.url)
}
val urlNode: Node = XpathUtils.getAsNode(document, IMG_XPATH)
?: throw HostException(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return Pair(title, urlNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
@@ -1,29 +1,23 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.download.ImageDownloadRunnable
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
import org.w3c.dom.Node
import java.util.*
class ViprImHost(
httpService: HTTPService,
dataTransaction: DataTransaction,
downloadSpeedService: DownloadSpeedService,
) : Host("vipr.im", 15, httpService, dataTransaction, downloadSpeedService) {
private val log by me.vripper.delegate.LoggerDelegate()
internal class ViprImHost : Host("vipr.im", 15) {
private val log by LoggerDelegate()
@Throws(HostException::class)
override fun resolve(
url: String,
document: Document,
context: ImageDownloadContext
context: ImageDownloadRunnable.Context
): Pair<String, String> {
val document = fetchDocument(context.imageEntity.url, context)
val imgNode: Node = try {
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url))
XpathUtils.getAsNode(document, IMG_XPATH)
} catch (e: XpathException) {
throw HostException(e)
@@ -31,11 +25,11 @@ class ViprImHost(
String.format(
"Xpath '%s' cannot be found in '%s'",
IMG_XPATH,
url
context.imageEntity.url
)
)
return try {
log.debug(String.format("Resolving name and image url for %s", url))
log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url))
val imgTitle =
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
.map { obj: Node -> obj.textContent }
@@ -0,0 +1,55 @@
package me.vripper.listeners
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.services.*
import me.vripper.services.download.DownloadService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
object AppManager : KoinComponent {
private val eventBus: EventBus by inject()
private val dataAccessService: DataAccessService by inject()
private val metadataService: MetadataService by inject()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
private val downloadSpeedService: DownloadSpeedService by inject()
private val httpService: HTTPService by inject()
private val retryPolicyService: RetryPolicyService by inject()
private val threadCacheService: ThreadCacheService by inject()
private val downloadService: DownloadService by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private var job: Job? = null
fun start() {
job?.cancel()
job = coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
httpService.client.close()
httpService.buildRequestConfig(it.settings.connectionSettings.timeout.toLong())
httpService.buildConnectionConfig(it.settings.connectionSettings.timeout.toLong())
httpService.buildConnectionPool()
httpService.buildClientBuilder()
vgAuthService.authenticate(it.settings)
retryPolicyService.maxAttempts = it.settings.connectionSettings.maxAttempts
threadCacheService.invalidate()
}
}
dataAccessService.setDownloadingToStopped()
dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted()
settingsService.init()
metadataService.fetchExisting()
downloadSpeedService.init()
downloadService.init()
}
fun stop() {
job?.cancel()
downloadService.halt()
downloadService.stop()
downloadSpeedService.halt()
}
}
@@ -1,16 +0,0 @@
package me.vripper.listeners
import me.vripper.services.DataTransaction
import me.vripper.services.MetadataService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
open class OnStartupListener : KoinComponent {
private val dataTransaction: DataTransaction by inject()
private val metadataService: MetadataService by inject()
open fun run() {
dataTransaction.setDownloadingToStopped()
metadataService.init()
}
}
@@ -1,3 +1,6 @@
package me.vripper.model
import kotlinx.serialization.Serializable
@Serializable
data class DownloadSpeed(val speed: Long)
@@ -1,3 +1,6 @@
package me.vripper.model
import kotlinx.serialization.Serializable
@Serializable
data class ErrorCount(val count: Int)
@@ -1,10 +1,13 @@
package me.vripper.model
import me.vripper.entities.domain.Status
import kotlinx.serialization.Contextual
import kotlinx.serialization.Serializable
import me.vripper.entities.Status
import java.time.LocalDateTime
import kotlin.io.path.Path
import kotlin.io.path.pathString
@Serializable
data class Post(
val id: Long = -1,
val postTitle: String,
@@ -17,18 +20,15 @@ data class Post(
val total: Int,
val hosts: Set<String>,
val downloadDirectory: String,
val addedOn: LocalDateTime = LocalDateTime.now(),
@Contextual val addedOn: LocalDateTime,
var folderName: String,
var status: Status = Status.STOPPED,
var done: Int = 0,
var rank: Int = Int.MAX_VALUE,
var size: Long = -1,
var downloaded: Long = 0,
val previews: List<String>,
val postedBy: String,
val resolvedNames: List<String>
) {
fun getDownloadFolder(): String {
return Path(downloadDirectory, folderName).pathString
}
fun getDownloadFolder() = Path(downloadDirectory, folderName).pathString
}
@@ -1,6 +1,10 @@
package me.vripper.model
import kotlinx.serialization.Serializable
@Serializable
data class QueueState(
val running: Int,
val remaining: Int
val remaining: Int,
val rank: List<Rank> = emptyList()
)
@@ -0,0 +1,6 @@
package me.vripper.model
import kotlinx.serialization.Serializable
@Serializable
data class Rank(val postEntityId: Long, val rank: Long)
@@ -1,6 +1,9 @@
package me.vripper.model
import kotlinx.serialization.Serializable
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import java.nio.file.Files
import kotlin.io.path.pathString
@Serializable
data class Settings(
@@ -17,11 +20,13 @@ data class ViperSettings(
val password: String = "",
val thanks: Boolean = false,
val host: String = "https://vipergirls.to",
val requestLimit: Long = 2,
val fetchMetadata: Boolean = false,
)
@Serializable
data class DownloadSettings(
val downloadPath: String = System.getProperty("user.home"),
val downloadPath: String = VRIPPER_DIR.resolve("downloads").also { Files.createDirectories(it) }.pathString,
val autoStart: Boolean = true,
val autoQueueThreshold: Int = 1,
val forceOrder: Boolean = false,
@@ -41,7 +46,7 @@ data class ConnectionSettings(
@Serializable
data class SystemSettings(
val tempPath: String = System.getProperty("java.io.tmpdir"),
val tempPath: String = VRIPPER_DIR.resolve("temp").also { Files.createDirectories(it) }.pathString,
val enableClipboardMonitoring: Boolean = false,
val clipboardPollingRate: Int = 500,
val maxEventLog: Int = 1_000,
@@ -1,5 +0,0 @@
package me.vripper.parser
import me.vripper.host.Host
data class ImageItem(val mainLink: String, val thumbLink: String, val host: Host)
@@ -1,74 +0,0 @@
package me.vripper.parser
import me.vripper.entities.LogEntryEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.PostParseException
import me.vripper.services.*
import me.vripper.utilities.Tasks
import me.vripper.utilities.formatToString
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.HttpClient
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import javax.xml.parsers.SAXParserFactory
class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
private val log by me.vripper.delegate.LoggerDelegate()
private val settingsService: SettingsService by inject()
private val retryPolicyService: RetryPolicyService by inject()
private val httpService: HTTPService by inject()
private val vgAuthService: VGAuthService by inject()
private val dataTransaction: DataTransaction by inject()
@Throws(PostParseException::class)
fun parse(): ThreadItem? {
log.debug("Parsing post $postId")
val httpGet =
HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
it.setParameter(
"p", postId.toString()
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
log.error(
"parsing failed for thread $threadId, post $postId", it.failure
)
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.POST, status = LogEntryEntity.Status.ERROR, message = """
Failed to process thread $threadId, post $postId
${it.failure.formatToString()}
""".trimIndent()
)
)
}.get(CheckedSupplier {
val connection: HttpClient = httpService.client
connection.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
factory.newSAXParser()
.parse(response.entity.content, threadLookupAPIResponseHandler)
threadLookupAPIResponseHandler.result
}
})
} catch (e: Exception) {
throw PostParseException(e)
} finally {
Tasks.decrement()
}
}
companion object {
private val factory = SAXParserFactory.newInstance()
}
}
@@ -1,78 +0,0 @@
package me.vripper.parser
import me.vripper.entities.LogEntryEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.PostParseException
import me.vripper.services.*
import me.vripper.utilities.Tasks
import me.vripper.utilities.formatToString
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.BufferedInputStream
import java.util.*
import javax.xml.parsers.SAXParserFactory
class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
private val log by me.vripper.delegate.LoggerDelegate()
private val cm: HTTPService by inject()
private val retryPolicyService: RetryPolicyService by inject()
private val vgAuthService: VGAuthService by inject()
private val settingsService: SettingsService by inject()
private val dataTransaction: DataTransaction by inject()
@Throws(PostParseException::class)
fun parse(): ThreadItem {
log.debug("Parsing thread $threadId")
val httpGet =
HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/vr.php").also {
it.setParameter(
"t",
threadId.toString()
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
log.error(
"parsing failed for thread $threadId",
it.failure
)
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.THREAD, status = LogEntryEntity.Status.ERROR, message = """
Failed to process thread $threadId
${it.failure.formatToString()}
""".trimIndent()
)
)
}.get(CheckedSupplier {
cm.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
factory.newSAXParser().parse(
BufferedInputStream(response.entity.content),
threadLookupAPIResponseHandler
)
threadLookupAPIResponseHandler.result
}
})
} catch (e: Exception) {
throw PostParseException(e)
} finally {
Tasks.decrement()
}
}
companion object {
private val factory = SAXParserFactory.newInstance()
}
}
@@ -1,20 +0,0 @@
package me.vripper.repositories
import me.vripper.entities.ImageEntity
import java.util.*
interface ImageRepository {
fun save(imageEntity: ImageEntity): ImageEntity
fun save(imageEntityList: List<ImageEntity>)
fun deleteAllByPostId(postId: Long)
fun findByPostId(postId: Long): List<ImageEntity>
fun countError(): Int
fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity>
fun stopByPostIdAndIsNotCompleted(postId: Long): Int
fun stopByPostIdAndIsNotCompleted(): Int
fun findByPostIdAndIsError(postId: Long): List<ImageEntity>
fun findById(id: Long): Optional<ImageEntity>
fun update(imageEntity: ImageEntity)
fun update(imageEntities: List<ImageEntity>)
fun deleteAllByPostId(postIds: List<Long>)
}
@@ -1,14 +0,0 @@
package me.vripper.repositories
import me.vripper.entities.LogEntryEntity
import java.util.*
interface LogRepository {
fun save(logEntryEntity: LogEntryEntity): LogEntryEntity
fun update(logEntryEntity: LogEntryEntity)
fun findAll(): List<LogEntryEntity>
fun findById(id: Long): Optional<LogEntryEntity>
fun deleteOldest(): List<Long>
fun deleteAll()
}
@@ -1,11 +0,0 @@
package me.vripper.repositories
import me.vripper.entities.MetadataEntity
import java.util.*
interface MetadataRepository {
fun save(metadataEntity: MetadataEntity): MetadataEntity
fun findByPostId(postId: Long): Optional<MetadataEntity>
fun deleteByPostId(postId: Long): Int
fun deleteAllByPostId(postIds: List<Long>)
}
@@ -1,74 +0,0 @@
package me.vripper.repositories.impl
import me.vripper.entities.LogEntryEntity
import me.vripper.repositories.LogRepository
import me.vripper.services.SettingsService
import me.vripper.tables.LogTable
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.notInSubQuery
import java.util.*
class LogRepositoryImpl(val settingsService: SettingsService) : LogRepository {
@Synchronized
override fun save(logEntryEntity: LogEntryEntity): LogEntryEntity {
val id = LogTable.insertAndGetId {
it[type] = logEntryEntity.type.name
it[status] = logEntryEntity.status.name
it[time] = logEntryEntity.time
it[message] = logEntryEntity.message.take(500)
}.value
return logEntryEntity.copy(id = id)
}
override fun update(logEntryEntity: LogEntryEntity) {
LogTable.update({ LogTable.id eq logEntryEntity.id }) {
it[type] = logEntryEntity.type.name
it[status] = logEntryEntity.status.name
it[message] = logEntryEntity.message.take(500)
}
}
override fun findById(id: Long): Optional<LogEntryEntity> {
val result = LogTable.select {
LogTable.id eq id
}.map(::transform)
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
}
override fun findAll(): List<LogEntryEntity> {
return LogTable.selectAll().map(::transform)
}
override fun deleteOldest(): List<Long> {
val sub = LogTable.slice(LogTable.id).selectAll().orderBy(LogTable.time to SortOrder.DESC)
.limit(settingsService.settings.systemSettings.maxEventLog)
val ids = LogTable.select { LogTable.id notInSubQuery sub }.map { it[LogTable.id].value }
LogTable.deleteWhere { LogTable.id notInSubQuery sub }
return ids
}
override fun deleteAll() {
LogTable.deleteAll()
}
private fun transform(resultRow: ResultRow): LogEntryEntity {
val id = resultRow[LogTable.id].value
val type = LogEntryEntity.Type.valueOf(resultRow[LogTable.type])
val status = LogEntryEntity.Status.valueOf(resultRow[LogTable.status])
val time = resultRow[LogTable.time]
val message = resultRow[LogTable.message]
return LogEntryEntity(
id,
type,
status,
time,
message
)
}
}
@@ -1,20 +1,27 @@
package me.vripper.services
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.FlowPreview
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.time.sample
import me.vripper.download.DownloadService
import me.vripper.entities.LogEntryEntity
import me.vripper.entities.PostEntity
import kotlinx.serialization.json.Json
import me.vripper.entities.*
import me.vripper.event.*
import me.vripper.exception.PostParseException
import me.vripper.model.*
import me.vripper.tasks.AddPostRunnable
import me.vripper.tasks.ThreadLookupRunnable
import me.vripper.services.download.DownloadService
import me.vripper.services.download.MovePosition
import me.vripper.services.download.QueueManager
import me.vripper.tasks.AddPostTask
import me.vripper.tasks.ThreadLookupTask
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.GLOBAL_EXECUTOR
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.taskRunner
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
import java.sql.DriverManager
import java.time.Duration
import java.util.concurrent.CompletableFuture
import java.util.concurrent.locks.ReentrantLock
import java.util.regex.Pattern
import kotlin.concurrent.withLock
@@ -22,15 +29,18 @@ import kotlin.io.path.Path
import kotlin.io.path.exists
import kotlin.jvm.optionals.getOrNull
class AppEndpointService(
@OptIn(ExperimentalCoroutinesApi::class, FlowPreview::class)
internal class AppEndpointService(
private val downloadService: DownloadService,
private val dataTransaction: DataTransaction,
private val queueManager: QueueManager,
private val dataAccessService: DataAccessService,
private val threadCacheService: ThreadCacheService,
private val settingsService: SettingsService,
private val vgAuthService: VGAuthService,
private val logService: LogService,
) : IAppEndpointService {
private val log by me.vripper.delegate.LoggerDelegate()
private val log by LoggerDelegate()
private val lock = ReentrantLock()
override suspend fun scanLinks(postLinks: String) {
@@ -38,10 +48,10 @@ class AppEndpointService(
if (postLinks.isBlank()) {
return
}
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isEmpty() }.map { it.trim() }
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isBlank() }.map { it.trim() }
.filter { it.isNotEmpty() }
for (link in urlList) {
log.debug("Starting to process thread: $link")
log.debug("Scanning: $link")
if (!link.startsWith(settingsService.settings.viperSettings.host)) {
continue
}
@@ -54,69 +64,60 @@ class AppEndpointService(
threadId = m.group(1).toLong()
postId = m.group(4)?.toLong()
if (postId == null) {
CompletableFuture.runAsync(
ThreadLookupRunnable(
taskRunner.submit(
ThreadLookupTask(
threadId, settingsService.settings
), GLOBAL_EXECUTOR
)
)
} else {
CompletableFuture.runAsync(
AddPostRunnable(
taskRunner.submit(
AddPostTask(
listOf(ThreadPostId(threadId, postId))
), GLOBAL_EXECUTOR
)
)
}
} else {
log.error("Cannot retrieve thread id from URL $link")
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.SCAN,
status = LogEntryEntity.Status.ERROR,
message = "Invalid link $link, link is missing the threadId"
)
)
log.error("Invalid link $link, link is missing the threadId")
continue
}
}
}
}
override suspend fun restartAll(posIds: List<Long>) {
override suspend fun restartAll(postEntityIds: List<Long>) {
lock.withLock {
downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
.map { it.get() })
downloadService.restartAll(postEntityIds.filter { dataAccessService.exists(it) }
.map { dataAccessService.findPostByEntityId(it) })
}
}
override suspend fun download(posts: List<ThreadPostId>) {
CompletableFuture.runAsync(
AddPostRunnable(posts), GLOBAL_EXECUTOR
)
taskRunner.submit(AddPostTask(posts))
}
override suspend fun stopAll(postIdList: List<Long>) {
override suspend fun stopAll(postEntityIds: List<Long>) {
lock.withLock {
downloadService.stop(postIdList)
downloadService.stop(postEntityIds)
}
}
override suspend fun remove(postIdList: List<Long>) {
override suspend fun remove(postEntityIds: List<Long>) {
lock.withLock {
downloadService.stop(postIdList)
dataTransaction.removeAll(postIdList)
downloadService.stop(postEntityIds)
dataAccessService.removeAll(postEntityIds)
}
}
override suspend fun clearCompleted(): List<Long> {
lock.withLock {
return dataTransaction.clearCompleted()
return dataAccessService.clearCompleted()
}
}
override suspend fun grab(threadId: Long): List<PostSelection> {
lock.withLock {
return try {
val thread = dataTransaction.findThreadByThreadId(threadId).orElseThrow {
val thread = dataAccessService.findThreadByThreadId(threadId).orElseThrow {
PostParseException(
String.format(
"Unable to find links for threadId = %s", threadId
@@ -124,6 +125,9 @@ class AppEndpointService(
)
}
val threadLookupResult = threadCacheService[thread.threadId]
if (threadLookupResult.error.isNotBlank()) {
throw PostParseException("Error occurred for threadId = $threadId, ${threadLookupResult.error}")
}
threadLookupResult.postItemList.map { postItem ->
PostSelection(
postItem.threadId,
@@ -137,23 +141,13 @@ class AppEndpointService(
postItem.forum,
postItem.imageItemList.map { it.thumbLink })
}.ifEmpty {
log.error(
String.format(
"Failed to get links for threadId = %s", threadId
)
)
throw PostParseException(
String.format(
"Failed to get links for threadId = %s", threadId
)
"Nothing found for threadId = $threadId"
)
}
} catch (e: Exception) {
throw PostParseException(
String.format(
"Failed to get links for threadId = %s, %s", threadId, e.message
)
)
log.error(e.message)
throw e
}
}
}
@@ -161,47 +155,46 @@ class AppEndpointService(
override suspend fun threadRemove(threadIdList: List<Long>) {
lock.withLock {
threadIdList.forEach {
dataTransaction.removeThread(it)
dataAccessService.removeThread(it)
}
}
}
override suspend fun threadClear() {
lock.withLock {
dataTransaction.clearQueueLinks()
dataAccessService.clearQueueLinks()
}
}
override suspend fun logClear() {
lock.withLock {
dataTransaction.deleteAllLogs()
}
}
override suspend fun renameToFirst(postIds: List<Long>) {
postIds.forEach { postId ->
dataTransaction
.findMetadataByPostId(postId)
override suspend fun renameToFirst(postEntityIds: List<Long>) {
postEntityIds.forEach { postEntityId ->
dataAccessService
.findMetadataByPostEntityId(postEntityId)
.map { it.data.resolvedNames }
.filter { it.isNotEmpty() }
.getOrNull()?.let { rename(postId, it.first()) }
.getOrNull()?.let { rename(postEntityId, it.first()) }
}
}
override suspend fun rename(postId: Long, newName: String) {
CompletableFuture.runAsync({
synchronized(postId.toString().intern()) {
dataTransaction.findPostByPostId(postId).ifPresent { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
)
override suspend fun rename(postEntityId: Long, newName: String) {
taskRunner.submit {
synchronized(postEntityId.toString().intern()) {
if (dataAccessService.exists(postEntityId)) {
dataAccessService.findPostByEntityId(postEntityId).let { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataAccessService.findImagesByPostEntityId(postEntityId),
post.downloadDirectory,
post.folderName,
newName
)
}
post.folderName = PathUtils.sanitize(newName)
dataAccessService.updatePost(post)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
}
}
}, GLOBAL_EXECUTOR)
}
}
override fun onNewPosts(): Flow<Post> =
@@ -215,7 +208,7 @@ class AppEndpointService(
override fun onDeletePosts() =
EventBus.events.filterIsInstance(PostDeleteEvent::class).flatMapConcat { it.postIds.asFlow() }
EventBus.events.filterIsInstance(PostDeleteEvent::class).flatMapConcat { it.postEntityIds.asFlow() }
override fun onUpdateMetadata() =
@@ -223,12 +216,12 @@ class AppEndpointService(
override suspend fun findAllPosts(): List<Post> {
return dataTransaction.findAllPosts().map(::mapper)
return dataAccessService.findAllPosts().map(::mapper)
}
private fun mapper(postEntity: PostEntity): Post {
val metadata: Metadata? = dataTransaction.findMetadataByPostId(postEntity.postId).orElse(null)
val images = dataTransaction.findImagesByPostId(postEntity.postId)
val metadata: Metadata? = dataAccessService.findMetadataByPostEntityId(postEntity.id).orElse(null)
val images = dataAccessService.findImagesByPostEntityId(postEntity.id)
return Post(
postEntity.id,
postEntity.postTitle,
@@ -236,8 +229,8 @@ class AppEndpointService(
postEntity.forum,
postEntity.url,
postEntity.token,
postEntity.postId,
postEntity.threadId,
postEntity.vgPostId,
postEntity.vgThreadId,
postEntity.total,
postEntity.hosts,
postEntity.downloadDirectory,
@@ -245,7 +238,6 @@ class AppEndpointService(
postEntity.folderName,
postEntity.status,
postEntity.done,
postEntity.rank,
postEntity.size,
postEntity.downloaded,
images.take(4).map { it.thumbUrl },
@@ -254,39 +246,28 @@ class AppEndpointService(
)
}
override suspend fun findPost(postId: Long): Post {
return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
override suspend fun findPost(postEntityId: Long): Post {
return mapper(dataAccessService.findPostByEntityId(postEntityId))
}
override suspend fun findImagesByPostId(postId: Long): List<Image> {
return dataTransaction.findImagesByPostId(postId)
override suspend fun findImagesByPostEntityId(postEntityId: Long): List<Image> {
return dataAccessService.findImagesByPostEntityId(postEntityId)
}
override fun onUpdateImages(postId: Long): Flow<Image> = EventBus.events.filterIsInstance(ImageEvent::class).map {
it.imageEntities.filter { imageEntity: Image -> imageEntity.postId == postId }
}.filter { it.isNotEmpty() }.flatMapConcat { it.asFlow() }
override fun onUpdateImagesByPostEntityId(postEntityId: Long): Flow<Image> =
EventBus.events.filterIsInstance(ImageEvent::class).map {
it.imageEntities.filter { imageEntity: Image -> imageEntity.postEntityId == postEntityId }
}.filter { it.isNotEmpty() }.flatMapConcat { it.asFlow() }
override fun onUpdateImages(): Flow<Image> =
EventBus.events.filterIsInstance(ImageEvent::class).flatMapConcat { it.imageEntities.asFlow() }
override fun onStopped(): Flow<Long> =
EventBus.events.filterIsInstance(StoppedEvent::class).flatMapConcat { it.postIds.asFlow() }
override suspend fun findAllLogs(): List<LogEntry> {
return dataTransaction.findAllLogs()
}
override fun onNewLog() = EventBus.events.filterIsInstance(LogCreateEvent::class).map { it.logEntryEntity }
override fun onUpdateLog() = EventBus.events.filterIsInstance(LogUpdateEvent::class).map {
it.logEntryEntity
}
override fun onDeleteLogs(): Flow<Long> = EventBus.events.filterIsInstance(LogDeleteEvent::class).flatMapConcat {
it.deleted.asFlow()
}
override suspend fun initLogger() = logService.init()
override fun onNewThread(): Flow<Thread> =
EventBus.events.filterIsInstance(ThreadCreateEvent::class).map { it.threadEntity }
@@ -304,11 +285,11 @@ class AppEndpointService(
override suspend fun findAllThreads(): List<Thread> {
return dataTransaction.findAllThreads()
return dataAccessService.findAllThreads()
}
override fun onDownloadSpeed(): Flow<Long> =
EventBus.events.filterIsInstance(DownloadSpeedEvent::class).map { it.downloadSpeed.speed }
override fun onDownloadSpeed(): Flow<DownloadSpeed> =
EventBus.events.filterIsInstance(DownloadSpeedEvent::class).map { it.downloadSpeed }
override fun onVGUserUpdate(): Flow<String> =
EventBus.events.filterIsInstance(VGUserLoginEvent::class).map { it.username }
@@ -316,8 +297,12 @@ class AppEndpointService(
override fun onQueueStateUpdate(): Flow<QueueState> =
EventBus.events.filterIsInstance(QueueStateEvent::class).map { it.queueState }
override fun onErrorCountUpdate(): Flow<Int> =
EventBus.events.filterIsInstance(ErrorCountEvent::class).map { it.errorCount.count }
override suspend fun getQueueState(): QueueState {
return queueManager.getQueueState()
}
override fun onErrorCountUpdate(): Flow<ErrorCount> =
EventBus.events.filterIsInstance(ErrorCountEvent::class).map { it.errorCount }
override fun onTasksRunning(): Flow<Boolean> =
EventBus.events.filterIsInstance(LoadingTasks::class).sample(Duration.ofMillis(500)).map { it.loading }
@@ -335,4 +320,152 @@ class AppEndpointService(
override suspend fun getVersion(): String = ApplicationProperties.VERSION
override suspend fun move(postEntityId: Long, position: MovePosition) {
downloadService.move(postEntityId, position)
}
override suspend fun dbMigration(): String {
val conn = try {
DriverManager
.getConnection("jdbc:h2:file:$VRIPPER_DIR/vripper;DB_CLOSE_DELAY=-1;IFEXISTS=TRUE")
} catch (_: JdbcSQLNonTransientConnectionException) {
return "Old database not found, nothing to do"
}
var postsCount = 0
var threadCount = 0
conn.use { conn ->
conn.prepareStatement("select * from post").use { statement ->
statement.executeQuery().use { resultSet ->
while (resultSet.next()) {
val done = resultSet.getInt("DONE")
val hosts = resultSet.getString("HOSTS")
val outputPath = resultSet.getString("OUTPUT_PATH")
val postId = resultSet.getLong("POST_ID")
val status = resultSet.getString("STATUS")
val threadId = resultSet.getLong("THREAD_ID")
val postTitle = resultSet.getString("POST_TITLE")
val threadTitle = resultSet.getString("THREAD_TITLE")
val forum = resultSet.getString("FORUM")
val total = resultSet.getInt("TOTAL")
val size = resultSet.getLong("SIZE")
val downloaded = resultSet.getLong("DOWNLOADED")
val url = resultSet.getString("URL")
val token = resultSet.getString("TOKEN")
val addedAt = resultSet.getTimestamp("ADDED_AT")
val folderName = resultSet.getString("FOLDER_NAME") ?: ""
val exists = dataAccessService.existsPostId(postId)
if (exists) {
continue
}
val post = PostEntity(
postTitle = postTitle,
threadTitle = threadTitle,
forum = forum,
url = url,
token = token,
vgPostId = postId,
vgThreadId = threadId,
total = total,
hosts = hosts.split(";").dropLastWhile { it.isEmpty() }.toSet(),
downloadDirectory = outputPath,
addedOn = addedAt.toLocalDateTime(),
folderName = folderName,
status = Status.valueOf(status),
done = done,
size = size,
downloaded = downloaded,
)
val images = mutableListOf<ImageEntity>()
// load images
conn.prepareStatement("select * from image where post_id = ?").use { statement ->
statement.setLong(1, postId)
statement.executeQuery().use { set ->
while (set.next()) {
val downloaded = set.getLong("DOWNLOADED")
val host = set.getByte("HOST")
val index = set.getInt("INDEX")
val status = Status.valueOf(set.getString("STATUS"))
val size = set.getLong("SIZE")
val url = set.getString("URL")
val thumbUrl = set.getString("THUMB_URL")
val fileName = set.getString("FILENAME") ?: ""
ImageEntity(
url = url,
thumbUrl = thumbUrl,
host = host,
index = index,
size = size,
downloaded = downloaded,
status = status,
filename = fileName,
).also { images.add(it) }
}
}
}
val savedPost = dataAccessService.saveAndNotify(post, images)
//load meta
conn.prepareStatement("select * from metadata where post_id = ?").use { statement ->
statement.setLong(1, postId)
statement.executeQuery().use { set ->
if (set.next()) {
val data = Json.decodeFromString(set.getString("DATA")) as MetadataEntity.Data
val metadata = MetadataEntity(
postIdRef = savedPost.id,
data = data
)
dataAccessService.saveMetadata(metadata)
}
}
}
postsCount++
}
}
}
conn.prepareStatement("select * from thread").use { statement ->
statement.executeQuery().use { set ->
while (set.next()) {
val total = set.getInt("TOTAL")
val url = set.getString("URL")
val threadId = set.getLong("THREAD_ID")
val title = set.getString("TITLE")
if (dataAccessService.findThreadByThreadId(threadId).isPresent) {
continue
}
val threadEntity = ThreadEntity(
title = title,
link = url,
threadId = threadId,
total = total,
)
dataAccessService.save(threadEntity)
threadCount++
}
}
}
}
return if (postsCount == 0 && threadCount == 0) {
"Nothing to do, everything was imported"
} else {
"Successfully imported $postsCount posts and $threadCount threads"
}
}
override fun connectionState(): String {
return ""
}
}
@@ -0,0 +1,297 @@
package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.entities.*
import me.vripper.event.*
import me.vripper.model.ErrorCount
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.PathUtils.sanitize
import me.vripper.vgapi.PostItem
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.*
import java.util.concurrent.TimeUnit
import kotlin.io.path.pathString
internal class DataAccessService(
private val settingsService: SettingsService,
private val postRepository: PostRepository,
private val imageRepository: ImageRepository,
private val threadRepository: ThreadRepository,
private val metadataRepository: MetadataRepository,
private val eventBus: EventBus,
) {
private val log by LoggerDelegate()
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findById(id) }
}
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
return transaction { postRepository.save(postEntities) }
}
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>): PostEntity {
val savedPost = transaction {
val savedPost =
postRepository.save(listOf(postEntity)).first()
save(images.map { it.copy(postEntityId = savedPost.id) })
// Publish event inside transaction for consistency
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
savedPost
}
return savedPost
}
fun updatePosts(postEntities: List<PostEntity>) {
transaction { postRepository.update(postEntities) }
postEntities.forEach { postEntity ->
postEntityIdCache.put(postEntity.id, postEntity)
}
// Log and publish event after transaction commit but before returning
log.debug("[{}] Publishing event: PostUpdateEvent for {} posts", System.currentTimeMillis(), postEntities.size)
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
fun updatePost(postEntity: PostEntity) {
transaction { postRepository.update(postEntity) }
postEntityIdCache.put(postEntity.id, postEntity)
// Log and publish event after transaction commit but before returning
log.debug("[{}] Publishing event: PostUpdateEvent for post {}", System.currentTimeMillis(), postEntity.id)
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
fun save(threadEntity: ThreadEntity) {
val savedThread = transaction { threadRepository.save(threadEntity) }
log.debug("[{}] Publishing event: ThreadCreateEvent for thread {}", System.currentTimeMillis(), savedThread.id)
eventBus.publishEvent(ThreadCreateEvent(savedThread))
}
fun update(threadEntity: ThreadEntity) {
transaction { threadRepository.update(threadEntity) }
log.debug("[{}] Publishing event: ThreadUpdateEvent for thread {}", System.currentTimeMillis(), threadEntity.id)
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
}
fun updateImages(imageEntities: List<ImageEntity>) {
transaction { imageRepository.update(imageEntities) }
log.debug("[{}] Publishing event: ImageEvent for {} images", System.currentTimeMillis(), imageEntities.size)
eventBus.publishEvent(ImageEvent(imageEntities))
}
fun updateImage(imageEntity: ImageEntity, persist: Boolean = true) {
if (persist) {
transaction {
imageRepository.update(imageEntity)
}
}
log.debug("[{}] Publishing event: ImageEvent for image {}", System.currentTimeMillis(), imageEntity.id)
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
}
fun exists(postEntityId: Long): Boolean {
if (postEntityIdCache.getIfPresent(postEntityId) != null) {
return true
}
return transaction { postRepository.existByPostEntityId(postEntityId) }
}
fun existsPostId(postId: Long): Boolean {
return transaction { postRepository.existByPostId(postId) }
}
@Synchronized
fun newPosts(postItems: List<PostItem>): List<PostEntity> {
val posts = postItems.associate { postItem ->
val postEntity = PostEntity(
postTitle = postItem.title,
url = postItem.url,
token = postItem.securityToken,
vgPostId = postItem.postId,
vgThreadId = postItem.threadId,
total = postItem.imageCount,
hosts = postItem.hosts.map { "${it.first} (${it.second})" }.toSet(),
threadTitle = postItem.threadTitle,
forum = postItem.forum,
downloadDirectory = PathUtils.calculateDownloadPath(
postItem.forum,
postItem.threadTitle,
settingsService.settings
).pathString,
folderName = if (settingsService.settings.downloadSettings.appendPostId) "${sanitize(postItem.title)}_${postItem.postId}" else sanitize(
postItem.title
)
)
val imageEntities = postItem.imageItemList.mapIndexed { index, imageItem ->
ImageEntity(
url = imageItem.mainLink,
thumbUrl = imageItem.thumbLink,
host = imageItem.host.hostId,
index = index,
)
}
Pair(postEntity, imageEntities)
}
val savedPosts = transaction {
val savedPosts = save(posts.keys.toList())
savedPosts.associateWith { postEntity ->
posts.entries.find { postItem -> postItem.key.vgPostId == postEntity.vgPostId }?.value ?: emptyList()
}.forEach { (key, value) ->
save(value.map { it.copy(postEntityId = key.id) })
}
savedPosts
}
eventBus.publishEvent(PostCreateEvent(savedPosts))
return savedPosts
}
private fun save(imageEntities: List<ImageEntity>) {
transaction { imageRepository.save(imageEntities) }
}
fun finishPost(id: Long, automatic: Boolean = false) {
val post = findPostByEntityId(id)
val imagesInErrorStatus = findByIdAndIsError(id)
if (imagesInErrorStatus.isNotEmpty()) {
post.status = Status.ERROR
updatePost(post)
} else {
if (post.done < post.total) {
post.status = Status.STOPPED
updatePost(post)
} else {
post.status = Status.FINISHED
transaction {
updatePost(post)
if (settingsService.settings.downloadSettings.clearCompleted && automatic) {
remove(listOf(post.id))
}
}
}
}
}
private fun findByIdAndIsError(postEntityId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostEntityIdAndIsError(postEntityId) }
}
private fun remove(postEntityIds: List<Long>) {
transaction {
metadataRepository.deleteAllByPostEntityId(postEntityIds)
imageRepository.deleteAllByPostEntityId(postEntityIds)
postRepository.deleteAll(postEntityIds)
}
postEntityIds.forEach { postEntityId ->
postEntityIdCache.get(postEntityId)?.let { postEntityIdCache.invalidate(it.id) }
postEntityIdCache.invalidate(postEntityId)
}
log.debug("[{}] Publishing event: PostDeleteEvent for {} posts", System.currentTimeMillis(), postEntityIds.size)
eventBus.publishEvent(PostDeleteEvent(postEntityIds = postEntityIds))
log.debug("[{}] Publishing event: ErrorCountEvent after post deletion", System.currentTimeMillis())
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
fun removeThread(threadId: Long) {
transaction { threadRepository.deleteByThreadId(threadId) }
log.debug("[{}] Publishing event: ThreadDeleteEvent for thread {}", System.currentTimeMillis(), threadId)
eventBus.publishEvent(ThreadDeleteEvent(threadId))
}
fun clearCompleted(): List<Long> {
val completed = transaction { postRepository.findCompleted() }
remove(completed)
return completed
}
fun removeAll(postEntityIds: List<Long> = emptyList()) {
if (postEntityIds.isNotEmpty()) {
remove(postEntityIds)
} else {
remove(findAllPosts().map(PostEntity::id))
}
}
fun stopImagesByPostEntityIdAndIsNotCompleted(postEntityId: Long) {
transaction { imageRepository.stopByPostEntityIdAndIsNotCompleted(postEntityId) }
}
fun stopImagesByPostEntityIdAndIsNotCompleted() {
transaction { imageRepository.stopByPostEntityIdAndIsNotCompleted() }
}
fun saveMetadata(metadataEntity: MetadataEntity) {
transaction { metadataRepository.save(metadataEntity) }
log.debug(
"[{}] Publishing event: MetadataUpdateEvent for post {}",
System.currentTimeMillis(),
metadataEntity.postIdRef
)
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
}
fun clearQueueLinks() {
transaction { threadRepository.deleteAll() }
log.debug("[{}] Publishing event: ThreadClearEvent", System.currentTimeMillis())
eventBus.publishEvent(ThreadClearEvent())
}
fun setDownloadingToStopped() {
transaction { postRepository.setDownloadingToStopped() }
}
fun findAllPosts(): List<PostEntity> {
return transaction { postRepository.findAll() }
}
fun findPostByEntityId(id: Long): PostEntity {
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
}
fun findImagesByPostEntityId(postEntityId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostEntityId(postEntityId) }
}
fun findImageById(id: Long): Optional<ImageEntity> {
return transaction { imageRepository.findById(id) }
}
fun findAllThreads(): List<ThreadEntity> {
return transaction { threadRepository.findAll() }
}
fun findThreadById(id: Long): Optional<ThreadEntity> {
return transaction {
threadRepository.findById(id)
}
}
fun findByPostEntityIdAndIsNotCompleted(postEntityId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostEntityIdAndIsNotCompleted(postEntityId) }
}
fun countImagesInError(): Int {
return transaction { imageRepository.countError() }
}
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
return transaction { threadRepository.findByThreadId(threadId) }
}
fun findAllNonCompletedPostEntityIds(): List<Long> {
return transaction { postRepository.findAllNonCompletedPostEntityIds() }
}
fun findMetadataByPostEntityId(postEntityId: Long): Optional<MetadataEntity> {
return transaction { metadataRepository.findByPostEntityId(postEntityId) }
}
}
@@ -1,319 +0,0 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.entities.*
import me.vripper.entities.domain.Status
import me.vripper.event.*
import me.vripper.model.ErrorCount
import me.vripper.parser.PostItem
import me.vripper.repositories.*
import me.vripper.utilities.PathUtils
import me.vripper.utilities.PathUtils.sanitize
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.*
import kotlin.io.path.pathString
class DataTransaction(
private val settingsService: SettingsService,
private val postDownloadStateRepository: PostDownloadStateRepository,
private val imageRepository: ImageRepository,
private val threadRepository: ThreadRepository,
private val metadataRepository: MetadataRepository,
private val logRepository: LogRepository,
private val eventBus: EventBus,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
return transaction { postDownloadStateRepository.save(postEntities) }
}
fun updatePosts(postEntities: List<PostEntity>) {
transaction { postDownloadStateRepository.update(postEntities) }
coroutineScope.launch {
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
}
fun updatePost(postEntity: PostEntity) {
transaction { postDownloadStateRepository.update(postEntity) }
coroutineScope.launch {
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
}
fun save(threadEntity: ThreadEntity) {
val savedThread = transaction { threadRepository.save(threadEntity) }
coroutineScope.launch {
eventBus.publishEvent(ThreadCreateEvent(savedThread))
}
}
fun update(threadEntity: ThreadEntity) {
transaction { threadRepository.update(threadEntity) }
coroutineScope.launch {
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
}
}
fun updateImages(imageEntities: List<ImageEntity>) {
transaction { imageRepository.update(imageEntities) }
coroutineScope.launch {
eventBus.publishEvent(ImageEvent(imageEntities))
}
}
fun updateImage(imageEntity: ImageEntity) {
transaction { imageRepository.update(imageEntity) }
coroutineScope.launch {
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
}
}
fun exists(postId: Long): Boolean {
return transaction { postDownloadStateRepository.existByPostId(postId) }
}
@Synchronized
fun newPosts(postItems: List<PostItem>): List<PostEntity> {
var queuePosition = transaction {
getQueuePosition()?.plus(1) ?: 0
}
val posts = postItems.associate { postItem ->
val postEntity = PostEntity(
postTitle = postItem.title,
url = postItem.url,
token = postItem.securityToken,
postId = postItem.postId,
threadId = postItem.threadId,
total = postItem.imageCount,
hosts = postItem.hosts.map { "${it.first} (${it.second})" }.toSet(),
threadTitle = postItem.threadTitle,
forum = postItem.forum,
rank = queuePosition++,
downloadDirectory = PathUtils.calculateDownloadPath(
postItem.forum,
postItem.threadTitle,
settingsService.settings
).pathString,
folderName = if (settingsService.settings.downloadSettings.appendPostId) "${sanitize(postItem.title)}_${postItem.postId}" else sanitize(
postItem.title
)
)
val imageEntities = postItem.imageItemList.mapIndexed { index, imageItem ->
ImageEntity(
postId = postItem.postId,
url = imageItem.mainLink,
thumbUrl = imageItem.thumbLink,
host = imageItem.host.hostId,
index = index,
)
}
Pair(postEntity, imageEntities)
}
val savedPosts = transaction {
val savedPosts = save(posts.keys.toList())
savedPosts.associateWith {
posts[it]!!
}.forEach { (key, value) ->
save(value.map { it.copy(postIdRef = key.id) })
}
savedPosts
}
coroutineScope.launch {
eventBus.publishEvent(PostCreateEvent(savedPosts))
}
return savedPosts
}
private fun getQueuePosition(): Int? {
return postDownloadStateRepository.findMaxRank()
}
private fun save(imageEntities: List<ImageEntity>) {
transaction { imageRepository.save(imageEntities) }
}
fun finishPost(postId: Long, automatic: Boolean = false) {
val post = findPostByPostId(postId).orElseThrow()
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
if (imagesInErrorStatus.isNotEmpty()) {
post.status = Status.ERROR
updatePost(post)
} else {
if (post.done < post.total) {
post.status = Status.STOPPED
updatePost(post)
} else {
post.status = Status.FINISHED
transaction {
updatePost(post)
if (settingsService.settings.downloadSettings.clearCompleted && automatic) {
remove(listOf(post.postId))
}
}
}
}
}
private fun findByPostIdAndIsError(postId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostIdAndIsError(postId) }
}
private fun remove(postIds: List<Long>) {
transaction {
metadataRepository.deleteAllByPostId(postIds)
imageRepository.deleteAllByPostId(postIds)
postDownloadStateRepository.deleteAll(postIds)
sortPostsByRank()
}
coroutineScope.launch {
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
}
coroutineScope.launch {
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
}
fun removeThread(threadId: Long) {
transaction { threadRepository.deleteByThreadId(threadId) }
coroutineScope.launch {
eventBus.publishEvent(ThreadDeleteEvent(threadId))
}
}
fun clearCompleted(): List<Long> {
val completed = transaction { postDownloadStateRepository.findCompleted() }
remove(completed)
return completed
}
fun removeAll(postIds: List<Long> = emptyList()) {
if (postIds.isNotEmpty()) {
remove(postIds)
} else {
remove(findAllPosts().map(PostEntity::postId))
}
}
fun stopImagesByPostIdAndIsNotCompleted(postId: Long) {
transaction { imageRepository.stopByPostIdAndIsNotCompleted(postId) }
}
fun saveMetadata(metadataEntity: MetadataEntity) {
transaction { metadataRepository.save(metadataEntity) }
coroutineScope.launch {
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
}
}
fun clearQueueLinks() {
transaction { threadRepository.deleteAll() }
coroutineScope.launch {
eventBus.publishEvent(ThreadClearEvent())
}
}
@Synchronized
fun sortPostsByRank() {
val postEntities = findAllPosts().sortedWith(Comparator.comparing(PostEntity::addedOn))
for (i in postEntities.indices) {
postEntities[i].rank = i
}
updatePosts(postEntities)
}
fun setDownloadingToStopped() {
transaction { postDownloadStateRepository.setDownloadingToStopped() }
}
fun findAllPosts(): List<PostEntity> {
return transaction { postDownloadStateRepository.findAll() }
}
fun findPostById(id: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findById(id) }
}
fun findImagesByPostId(postId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostId(postId) }
}
fun findImageById(id: Long): Optional<ImageEntity> {
return transaction { imageRepository.findById(id) }
}
fun findAllThreads(): List<ThreadEntity> {
return transaction { threadRepository.findAll() }
}
fun findThreadById(id: Long): Optional<ThreadEntity> {
return transaction {
threadRepository.findById(id)
}
}
fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity> {
return transaction { imageRepository.findByPostIdAndIsNotCompleted(postId) }
}
fun countImagesInError(): Int {
return transaction { imageRepository.countError() }
}
fun findPostByPostId(postId: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findByPostId(postId) }
}
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
return transaction { threadRepository.findByThreadId(threadId) }
}
fun saveLog(logEntryEntity: LogEntryEntity): LogEntryEntity {
val pair = transaction {
val saved = logRepository.save(logEntryEntity)
val deleted = logRepository.deleteOldest()
Pair(saved, deleted)
}
coroutineScope.launch {
eventBus.publishEvent(LogCreateEvent(pair.first))
}
coroutineScope.launch {
eventBus.publishEvent(LogDeleteEvent(pair.second))
}
return pair.first
}
fun updateLog(logEntryEntity: LogEntryEntity) {
transaction { logRepository.update(logEntryEntity) }
coroutineScope.launch {
eventBus.publishEvent(LogUpdateEvent(logEntryEntity))
}
}
fun deleteAllLogs() {
transaction { logRepository.deleteAll() }
}
fun findAllLogs(): List<LogEntryEntity> {
return transaction { logRepository.findAll() }
}
fun findAllNonCompletedPostIds(): List<Long> {
return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
}
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
return transaction { metadataRepository.findByPostId(postId) }
}
}
@@ -6,9 +6,10 @@ import me.vripper.event.DownloadSpeedEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.model.DownloadSpeed
import me.vripper.utilities.LoggerDelegate
import java.util.concurrent.atomic.AtomicLong
class DownloadSpeedService(
internal class DownloadSpeedService(
private val eventBus: EventBus,
) {
@@ -16,12 +17,15 @@ class DownloadSpeedService(
const val DOWNLOAD_POLL_RATE = 2500
}
private val log by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val bytesCount = AtomicLong(0)
private var job: Job? = null
private var queueStateUpdateJob: Job? = null
init {
coroutineScope.launch {
fun init() {
queueStateUpdateJob?.cancel()
queueStateUpdateJob = coroutineScope.launch {
eventBus.events.filterIsInstance(QueueStateEvent::class).collect {
if (it.queueState.running + it.queueState.remaining > 0) {
if (job == null || job?.isActive == false) {
@@ -30,8 +34,13 @@ class DownloadSpeedService(
while (isActive) {
delay(DOWNLOAD_POLL_RATE.toLong())
val newValue = bytesCount.getAndSet(0)
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))))
val speed = DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))
log.debug(
"[{}] Publishing event: DownloadSpeedEvent({})",
System.currentTimeMillis(),
speed
)
eventBus.publishEvent(DownloadSpeedEvent(speed))
}
}
}
@@ -39,6 +48,7 @@ class DownloadSpeedService(
job?.cancel()
coroutineScope.launch {
delay(DOWNLOAD_POLL_RATE + 500L)
log.debug("[{}] Publishing event: DownloadSpeedEvent(0)", System.currentTimeMillis())
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(0L)))
}
}
@@ -46,6 +56,11 @@ class DownloadSpeedService(
}
}
fun halt() {
queueStateUpdateJob?.cancel()
job?.cancel()
}
fun reportDownloadedBytes(count: Long) {
bytesCount.addAndGet(count)
}
@@ -1,100 +1,74 @@
package me.vripper.services
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.utilities.ApplicationProperties
import org.apache.hc.client5.http.config.ConnectionConfig
import org.apache.hc.client5.http.config.RequestConfig
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.cookie.StandardCookieSpec
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.client5.http.io.HttpClientConnectionManager
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
import org.apache.hc.core5.pool.PoolReusePolicy
import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
class HTTPService(
private val eventBus: EventBus,
settingsService: SettingsService
) {
internal class HTTPService {
companion object {
const val USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.0"
val cookieStore = BasicCookieStore()
}
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private lateinit var pcm: PoolingHttpClientConnectionManager
private lateinit var rc: RequestConfig
private lateinit var cc: ConnectionConfig
lateinit var client: CloseableHttpClient
private var connectionTimeout = settingsService.settings.connectionSettings.timeout
private var pcm: HttpClientConnectionManager = BasicHttpClientConnectionManager()
private var rc: RequestConfig = RequestConfig.DEFAULT
private var cc: ConnectionConfig = ConnectionConfig.DEFAULT
private var connectionExpiryJob: Job? = null
var client: CloseableHttpClient = HttpClients.createDefault()
init {
buildRequestConfig()
buildConnectionConfig()
buildConnectionPool()
buildClientBuilder()
coroutineScope.launch {
pcm.closeIdle(TimeValue.ofSeconds(60))
delay(15000)
}
coroutineScope.launch {
eventBus
.events
.filterIsInstance(SettingsUpdateEvent::class)
.collect {
if (connectionTimeout != it.settings.connectionSettings.timeout) {
connectionTimeout = it.settings.connectionSettings.timeout
client.close()
pcm.close()
buildRequestConfig()
buildConnectionConfig()
buildConnectionPool()
buildClientBuilder()
}
}
}
}
private fun buildConnectionPool() {
fun buildConnectionPool() {
pcm.close()
connectionExpiryJob?.cancel()
pcm = PoolingHttpClientConnectionManagerBuilder.create()
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
.setConnPoolPolicy(PoolReusePolicy.LIFO)
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
.setConnPoolPolicy(PoolReusePolicy.FIFO)
.setDefaultConnectionConfig(cc)
.setMaxConnTotal(Int.MAX_VALUE)
.setMaxConnPerRoute(Int.MAX_VALUE)
.build()
.build().also {
connectionExpiryJob = coroutineScope.launch {
while (isActive) {
it.closeExpired()
delay(60_000)
}
}
}
}
private fun buildRequestConfig() {
fun buildRequestConfig(connectionTimeout: Long) {
rc = RequestConfig.custom()
.setConnectionRequestTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setConnectionRequestTimeout(Timeout.ofSeconds(connectionTimeout))
.setCookieSpec(StandardCookieSpec.RELAXED)
.build()
}
private fun buildConnectionConfig() {
fun buildConnectionConfig(connectionTimeout: Long) {
cc = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setTimeToLive(TimeValue.ofMinutes(10))
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout))
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout))
.build()
}
private fun buildClientBuilder() {
fun buildClientBuilder() {
client = HttpClients.custom()
.setConnectionManager(pcm)
.setRedirectStrategy(DefaultRedirectStrategy.INSTANCE)
.setUserAgent(USER_AGENT)
.setUserAgent(ApplicationProperties.USER_AGENT)
.disableAutomaticRetries()
.setDefaultRequestConfig(rc)
.setDefaultCookieStore(cookieStore)
.build()
}
}
}
@@ -2,28 +2,26 @@ package me.vripper.services
import kotlinx.coroutines.flow.Flow
import me.vripper.model.*
import me.vripper.services.download.MovePosition
interface IAppEndpointService {
suspend fun scanLinks(postLinks: String)
suspend fun restartAll(posIds: List<Long> = listOf())
suspend fun remove(postIdList: List<Long>)
suspend fun stopAll(postIdList: List<Long> = emptyList())
suspend fun restartAll(postEntityIds: List<Long> = listOf())
suspend fun remove(postEntityIds: List<Long>)
suspend fun stopAll(postEntityIds: List<Long> = emptyList())
suspend fun clearCompleted(): List<Long>
suspend fun findPost(postId: Long): Post
suspend fun findPost(postEntityId: Long): Post
suspend fun findAllPosts(): List<Post>
suspend fun rename(postId: Long, newName: String)
suspend fun rename(postEntityId: Long, newName: String)
fun onNewPosts(): Flow<Post>
fun onUpdatePosts(): Flow<Post>
fun onDeletePosts(): Flow<Long>
fun onUpdateMetadata(): Flow<Metadata>
suspend fun findImagesByPostId(postId: Long): List<Image>
fun onUpdateImages(postId: Long): Flow<Image>
suspend fun findImagesByPostEntityId(postEntityId: Long): List<Image>
fun onUpdateImagesByPostEntityId(postEntityId: Long): Flow<Image>
fun onUpdateImages(): Flow<Image>
fun onStopped(): Flow<Long>
suspend fun logClear()
suspend fun findAllLogs(): List<LogEntry>
fun onNewLog(): Flow<LogEntry>
fun onUpdateLog(): Flow<LogEntry>
fun onDeleteLogs(): Flow<Long>
fun onNewThread(): Flow<Thread>
fun onUpdateThread(): Flow<Thread>
fun onDeleteThread(): Flow<Long>
@@ -33,10 +31,11 @@ interface IAppEndpointService {
suspend fun threadClear()
suspend fun grab(threadId: Long): List<PostSelection>
suspend fun download(posts: List<ThreadPostId>)
fun onDownloadSpeed(): Flow<Long>
fun onDownloadSpeed(): Flow<DownloadSpeed>
fun onVGUserUpdate(): Flow<String>
suspend fun getQueueState(): QueueState
fun onQueueStateUpdate(): Flow<QueueState>
fun onErrorCountUpdate(): Flow<Int>
fun onErrorCountUpdate(): Flow<ErrorCount>
fun onTasksRunning(): Flow<Boolean>
suspend fun getSettings(): Settings
suspend fun saveSettings(settings: Settings)
@@ -44,5 +43,9 @@ interface IAppEndpointService {
fun onUpdateSettings(): Flow<Settings>
suspend fun loggedInUser(): String
suspend fun getVersion(): String
suspend fun renameToFirst(postIds: List<Long>)
suspend fun renameToFirst(postEntityIds: List<Long>)
suspend fun dbMigration(): String
suspend fun initLogger()
suspend fun move(postEntityId: Long, position: MovePosition)
fun connectionState(): String
}
@@ -0,0 +1,49 @@
package me.vripper.services
import kotlinx.coroutines.*
import me.vripper.RingAppender
import me.vripper.entities.LogEntryEntity
import me.vripper.event.EventBus
import me.vripper.event.LogCreateEvent
import me.vripper.utilities.StackTraceElementUtils
import java.time.ZoneId
import java.time.format.DateTimeFormatter
class LogService(private val eventBus: EventBus) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
private var job: Job? = null
fun init() {
if (job != null) {
return
}
job = coroutineScope.launch {
while (isActive) {
while (!RingAppender.Static.events.isEmpty()) {
RingAppender.Static.events.poll()?.also { loggingEvent ->
eventBus.publishEvent(
LogCreateEvent(
LogEntryEntity(
loggingEvent.sequenceNumber,
loggingEvent.instant.atZone(ZoneId.systemDefault()).toLocalDateTime()
.format(dateTimeFormatter),
loggingEvent.threadName,
loggingEvent.loggerName,
loggingEvent.level.levelStr,
loggingEvent.formattedMessage,
loggingEvent.throwableProxy?.let { throwableProxy ->
StackTraceElementUtils.format(
throwableProxy
)
} ?: ""
)
)
)
}
}
delay(777)
}
}
}
}
@@ -1,113 +1,30 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.withPermit
import me.vripper.entities.MetadataEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.Tasks
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.net.URIBuilder
import org.w3c.dom.Node
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
import me.vripper.entities.PostEntity
import me.vripper.tasks.FetchMetadataTask
import me.vripper.utilities.taskRunner
class MetadataService(
private val httpService: HTTPService,
internal class MetadataService(
private val dataAccessService: DataAccessService,
private val settingsService: SettingsService,
private val dataTransaction: DataTransaction,
private val vgAuthService: VGAuthService,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
fun init() {
dataTransaction.findAllPosts().filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }
.map { it.postId }.forEach(::fetchMetadata)
}
fun fetchMetadata(postId: Long) {
Tasks.increment()
coroutineScope.launch {
RequestLimit.semaphore.withPermit {
try {
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postId.toString()
)
}.build())
val metadataEntity = httpService.client.execute(httpGet, vgAuthService.context) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
val document = HtmlUtils.clean(it.entity.content)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'$postId'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'$postId'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
}
try {
dataTransaction.saveMetadata(metadataEntity)
} catch (e: Exception) {
e.printStackTrace()
}
} finally {
Tasks.decrement()
}
}
}
}
private fun findTitleInContent(node: Node): List<String> {
val altTitle: MutableList<String> = mutableListOf()
findTitle(node, altTitle, AtomicBoolean(true))
return altTitle.stream().distinct().collect(Collectors.toList())
}
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
if (!keepGoing.get()) {
fun fetchExisting() {
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
if (node.nodeName == "a" || node.nodeName == "img") {
keepGoing.set(false)
dataAccessService.findAllPosts().filter { dataAccessService.findMetadataByPostEntityId(it.id).isEmpty }
.map { it }.forEach(::fetchMetadata)
}
fun fetchMetadata(post: PostEntity) {
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
if (node.nodeType == Node.ELEMENT_NODE) {
for (i in 0 until node.childNodes.length) {
val item = node.childNodes.item(i)
findTitle(item, altTitle, keepGoing)
if (!keepGoing.get()) {
return
}
}
} else if (node.nodeType == Node.TEXT_NODE) {
val text = node.textContent.trim()
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
text.lowercase().contains(e.lowercase())
}) {
altTitle.add(text)
}
}
taskRunner.submit(
FetchMetadataTask(
post
)
)
}
}
@@ -1,43 +1,17 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import net.jodah.failsafe.RetryPolicy
import net.jodah.failsafe.event.ExecutionAttemptedEvent
import dev.failsafe.RetryPolicy
import me.vripper.utilities.LoggerDelegate
import java.time.temporal.ChronoUnit
class RetryPolicyService(
val eventBus: EventBus, settingsService: SettingsService
) {
private val log by me.vripper.delegate.LoggerDelegate()
private var maxAttempts: Int = settingsService.settings.connectionSettings.maxAttempts
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
internal class RetryPolicyService {
private val log by LoggerDelegate()
var maxAttempts: Int = 3
init {
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
if (maxAttempts != it.settings.connectionSettings.maxAttempts) {
maxAttempts = it.settings.connectionSettings.maxAttempts
}
}
}
}
fun <T> buildRetryPolicyForDownload(): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
log.warn("#${it.attemptCount} tries failed", it.lastFailure)
}
}
fun <T> buildGenericRetryPolicy(): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
log.warn("#${e.attemptCount} tries failed", e.lastFailure)
}
fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
}.build()
}
}
@@ -1,9 +1,6 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
@@ -12,18 +9,18 @@ import me.vripper.event.SettingsUpdateEvent
import me.vripper.exception.ValidationException
import me.vripper.model.Settings
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import org.apache.commons.codec.digest.DigestUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.md5Hex
import java.io.FileWriter
import java.nio.file.*
import kotlin.io.path.readText
class SettingsService(private val eventBus: EventBus) {
private val log by me.vripper.delegate.LoggerDelegate()
private val log by LoggerDelegate()
private val configPath = VRIPPER_DIR.resolve("config.json")
private val customProxiesPath = VRIPPER_DIR.resolve("proxies.json")
private val proxies: MutableSet<String> = HashSet()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val json = Json {
encodeDefaults = true
@@ -32,19 +29,13 @@ class SettingsService(private val eventBus: EventBus) {
}
var settings = Settings()
init {
init()
}
private fun init() {
fun init() {
loadViperProxies()
restore()
coroutineScope.launch {
eventBus.publishEvent(SettingsUpdateEvent(settings))
}
eventBus.publishEvent(SettingsUpdateEvent(settings))
}
@OptIn(ExperimentalSerializationApi::class)
private fun loadViperProxies() {
try {
SettingsService::class.java.getResourceAsStream("/proxies.json")?.use {
@@ -57,7 +48,7 @@ class SettingsService(private val eventBus: EventBus) {
customProxiesPath.readText()
)
} catch (e: Exception) {
e.printStackTrace()
log.error("Failed to read custom proxies", e)
emptyList()
}
} else {
@@ -91,7 +82,7 @@ class SettingsService(private val eventBus: EventBus) {
check(settings)
val viperSettings = if (settings.viperSettings.login) {
if (this.settings.viperSettings.password != settings.viperSettings.password) {
settings.viperSettings.copy(password = DigestUtils.md5Hex(settings.viperSettings.password))
settings.viperSettings.copy(password = md5Hex(settings.viperSettings.password))
} else {
settings.viperSettings
}
@@ -100,9 +91,7 @@ class SettingsService(private val eventBus: EventBus) {
}
this.settings = settings.copy(viperSettings = viperSettings)
save()
coroutineScope.launch {
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
}
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
}
private fun restore() {
@@ -111,7 +100,7 @@ class SettingsService(private val eventBus: EventBus) {
settings = json.decodeFromString(configPath.readText())
}
} catch (e: Exception) {
log.error("Failed restore user settings", e)
log.error("Failed to restore user settings", e)
settings = Settings()
}
if (!proxies.contains(settings.viperSettings.host)) {
@@ -146,7 +135,7 @@ class SettingsService(private val eventBus: EventBus) {
fun check(settings: Settings) {
val path: Path = try {
Paths.get(settings.downloadSettings.downloadPath)
} catch (e: InvalidPathException) {
} catch (_: InvalidPathException) {
throw ValidationException(
String.format(
"%s is invalid", settings.downloadSettings.downloadPath
@@ -197,5 +186,11 @@ class SettingsService(private val eventBus: EventBus) {
"Invalid clipboard monitoring polling rate settings, values must be >= 500"
)
}
if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 5) {
throw ValidationException(
"Invalid request rate limit, values must be in [1,5]"
)
}
}
}
@@ -2,39 +2,28 @@ package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.sync.withPermit
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.parser.ThreadItem
import me.vripper.parser.ThreadLookupAPIParser
import me.vripper.utilities.RequestLimit
import java.util.*
import me.vripper.vgapi.ThreadItem
import me.vripper.vgapi.ThreadLookupAPIParser
import java.util.concurrent.ExecutionException
import java.util.concurrent.TimeUnit
class ThreadCacheService(val eventBus: EventBus) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
fun init() {
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
cache.invalidateAll()
}
}
}
internal class ThreadCacheService(val dataAccessService: DataAccessService) {
private val cache: LoadingCache<Long, ThreadItem> =
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
runBlocking {
RequestLimit.semaphore.withPermit {
ThreadLookupAPIParser(threadId).parse()
val threadItem = ThreadLookupAPIParser(threadId).parse()
dataAccessService.findThreadByThreadId(threadItem.threadId).ifPresent {
if (threadItem.postItemList.isNotEmpty()) {
dataAccessService.update(it.copy(total = threadItem.postItemList.size))
}
}
threadItem
}
fun invalidate() {
cache.invalidateAll()
}
@Throws(ExecutionException::class)
operator fun get(threadId: Long): ThreadItem {
return cache[threadId]
@@ -1,72 +1,58 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.entities.PostEntity
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.event.VGUserLoginEvent
import me.vripper.exception.VripperException
import me.vripper.model.Settings
import me.vripper.tasks.LeaveThanksRunnable
import me.vripper.utilities.GLOBAL_EXECUTOR
import me.vripper.tasks.LeaveThanksTask
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.taskRunner
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.cookie.Cookie
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.http.message.BasicNameValuePair
import java.util.concurrent.CompletableFuture
class VGAuthService(
internal class VGAuthService(
private val cm: HTTPService,
private val settingsService: SettingsService,
private val eventBus: EventBus
private val eventBus: EventBus,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val log by me.vripper.delegate.LoggerDelegate()
val context: HttpClientContext = HttpClientContext.create()
private val log by LoggerDelegate()
private val vgCookies: MutableList<Cookie> = mutableListOf()
var loggedUser = ""
private var authenticated = false
init {
context.cookieStore = BasicCookieStore()
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
authenticate(it.settings)
}
}
authenticate(settingsService.settings)
}
private fun authenticate(settings: Settings) {
authenticated = false
fun authenticate(settings: Settings) {
if (!settings.viperSettings.login) {
log.debug("Authentication option is disabled")
context.cookieStore.clear()
loggedUser = ""
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
synchronized(this) {
authenticated = false
loggedUser = ""
}
synchronized(vgCookies) {
vgCookies.clear()
}
log.debug("[{}] Publishing event: VGUserLoginEvent", System.currentTimeMillis())
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
val username = settings.viperSettings.username
val password = settings.viperSettings.password
if (username.isEmpty() || password.isEmpty()) {
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
context.cookieStore.clear()
loggedUser = ""
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
synchronized(this) {
authenticated = false
loggedUser = ""
}
synchronized(vgCookies) {
vgCookies.clear()
}
log.debug("[{}] Publishing event: VGUserLoginEvent (empty credentials)", System.currentTimeMillis())
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
val postAuth = HttpPost(settings.viperSettings.host + "/login.php?do=login").also {
@@ -78,13 +64,14 @@ class VGAuthService(
BasicNameValuePair("vb_login_md5password", password)
)
)
it.addHeader("Referer", settings.viperSettings.host)
it.addHeader(
"Host", settings.viperSettings.host.replace("https://", "").replace("http://", "")
)
it.setAbsoluteRequestUri(true)
}
log.info("Authenticating: {}", postAuth)
try {
val context = HttpClientContext.create().apply {
cookieStore =
BasicCookieStore()
}
cm.client.execute(postAuth, context) { response ->
if (response.code / 100 != 2) {
throw VripperException("Unexpected response code returned ${response.code}")
@@ -92,38 +79,79 @@ class VGAuthService(
val responseBody = EntityUtils.toString(response.entity)
log.debug("Authentication with ViperGirls response body:{}", responseBody)
}
if (context.cookieStore.cookies.stream().map { obj: Cookie -> obj.name }
.noneMatch { e: String -> e == "vg_userid" }) {
val userIdCookie = context.cookieStore.cookies.find { it.name == "vg_userid" }
val passwordCookie = context.cookieStore.cookies.find { it.name == "vg_password" }
if (userIdCookie == null || passwordCookie == null) {
log.error(
"Failed to authenticate user with {}, missing vg_userid cookie",
"Failed to authenticate user with {}, missing vg_userid/vg_password cookie",
settings.viperSettings.host
)
log.debug("[${System.currentTimeMillis()}] Publishing event: VGUserLoginEvent (missing cookies)")
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
} catch (e: Exception) {
context.cookieStore.clear()
loggedUser = ""
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
synchronized(vgCookies) {
vgCookies.clear()
vgCookies.add(BasicClientCookie(userIdCookie.name, userIdCookie.value).apply {
domain = userIdCookie.domain
})
vgCookies.add(BasicClientCookie(passwordCookie.name, passwordCookie.value).apply {
domain = passwordCookie.domain
})
}
} catch (e: Exception) {
log.error(
"Failed to authenticate user with " + settings.viperSettings.host, e
)
synchronized(this) {
authenticated = false
loggedUser = ""
}
log.debug("[{}] Publishing event: VGUserLoginEvent (exception)", System.currentTimeMillis())
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
authenticated = true
loggedUser = username
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
synchronized(this) {
authenticated = true
loggedUser = username
}
log.debug("[{}] Publishing event: VGUserLoginEvent for user {}", System.currentTimeMillis(), username)
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
log.info("Successfully logged in as: {}", loggedUser)
}
fun leaveThanks(postEntity: PostEntity) {
CompletableFuture.runAsync(
LeaveThanksRunnable(postEntity, authenticated, context), GLOBAL_EXECUTOR
val context = createVgContext()
taskRunner.submit(
LeaveThanksTask(postEntity, authenticated, context)
)
}
fun createVgContext(): HttpClientContext {
val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
synchronized(vgCookies) {
if (authenticated && vgCookies.isNotEmpty()) {
vgCookies.forEach { c -> context.cookieStore.addCookie(c) }
}
}
return context
}
fun createClickContext(): HttpClientContext {
val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
synchronized(vgCookies) {
if (authenticated && vgCookies.isNotEmpty()) {
vgCookies.forEach { c ->
context.cookieStore.addCookie(
BasicClientCookie(
c.name,
c.value
).apply { domain = "viper.click" })
}
}
}
return context
}
}
@@ -0,0 +1,171 @@
package me.vripper.services.download
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
import me.vripper.event.EventBus
import me.vripper.event.StoppedEvent
import me.vripper.services.DataAccessService
import me.vripper.services.SettingsService
import me.vripper.services.download.SharedLock.downloadManagerCondition
import me.vripper.services.download.SharedLock.downloadManagerLock
import me.vripper.utilities.LoggerDelegate
import org.jetbrains.exposed.sql.transactions.transaction
import kotlin.concurrent.withLock
import kotlin.math.min
internal class DownloadService(
private val settingsService: SettingsService,
private val dataAccessService: DataAccessService,
private val queueManager: QueueManager,
private val eventBus: EventBus
) {
private val maxPoolSize: Int = 24
private val log by LoggerDelegate()
private var downloadMonitorThread: Thread? = null
fun init() {
downloadMonitorThread?.interrupt()
downloadMonitorThread = Thread.ofVirtual().name("Download Monitor").unstarted(Runnable {
log.info("DownloadManager have been initialized")
while (!Thread.currentThread().isInterrupted) {
downloadManagerLock.withLock {
val candidates = getCandidates()
val canRunPerHost = canRun(candidates.keys)
val accepted = candidates.entries.flatMap {
it.value.take(canRunPerHost[it.key] ?: 0)
}
queueManager.accept(accepted)
try {
downloadManagerCondition.await()
} catch (_: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
log.info("Scheduler have been shutdown")
})
downloadMonitorThread?.start()
}
fun halt() {
downloadMonitorThread?.interrupt()
}
fun stop(postEntityIds: List<Long> = emptyList()) {
downloadManagerLock.withLock {
if (postEntityIds.isNotEmpty()) {
stopInternal(postEntityIds)
log.debug(
"[{}] Publishing event: StoppedEvent for {} posts",
System.currentTimeMillis(),
postEntityIds.size
)
eventBus.publishEvent(StoppedEvent(postEntityIds))
} else {
stopAll()
log.debug("[{}] Publishing event: StoppedEvent for all posts", System.currentTimeMillis())
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
}
}
fun restartAll(postEntities: List<PostEntity> = emptyList()) {
if (postEntities.isNotEmpty()) {
restart(postEntities.associateWith { dataAccessService.findByPostEntityIdAndIsNotCompleted(it.id) })
} else {
restart(
dataAccessService.findAllPosts()
.associateWith { dataAccessService.findByPostEntityIdAndIsNotCompleted(it.id) })
}
}
fun move(postEntityId: Long, position: MovePosition) {
queueManager.move(postEntityId, position)
}
private fun restart(posts: Map<PostEntity, List<ImageEntity>>) {
downloadManagerLock.withLock {
val toProcess = mutableMapOf<PostEntity, List<ImageEntity>>()
for ((post, images) in posts) {
if (images.isEmpty()) {
continue
}
if (queueManager.isPending(post.id)) {
continue
}
toProcess[post] = images
}
toProcess.forEach { (post, images) ->
post.status = Status.PENDING
images.forEach { image ->
with(image) {
this.status = Status.PENDING
this.downloaded = 0
}
}
}
transaction {
dataAccessService.updatePosts(
toProcess.keys.toList()
)
dataAccessService.updateImages(
toProcess.values.flatten()
)
}
toProcess.entries.sortedBy { it.key.addedOn }.forEach { (post, images) ->
images.map { image ->
log.debug("Enqueuing a job for ${post.url}")
ImageQueueElement(
image.id, image.postEntityId, image.host
)
}.also { queueManager.addPending(it) }
}
downloadManagerCondition.signal()
}
}
private fun stopAll() {
queueManager.clearPending()
queueManager.clearRunning()
dataAccessService.findAllNonCompletedPostEntityIds().forEach {
dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it)
dataAccessService.finishPost(it)
}
}
private fun stopInternal(postEntityIds: List<Long>) {
postEntityIds.forEach {
queueManager.clearPending(it)
queueManager.clearRunning(it)
dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it)
dataAccessService.finishPost(it)
}
}
private fun canRun(hosts: Set<Byte>): Map<Byte, Int> {
val runningSnapshot = queueManager.runningTotalCount().toMutableMap()
return hosts.associateWith {
val totalRunning = runningSnapshot.values.sum()
val canRunGlobally =
if (settingsService.settings.connectionSettings.maxGlobalConcurrent == 0) maxPoolSize - totalRunning else settingsService.settings.connectionSettings.maxGlobalConcurrent - totalRunning
val canRunPerHost =
settingsService.settings.connectionSettings.maxConcurrentPerHost - (runningSnapshot[it] ?: 0)
val min = min(canRunGlobally, canRunPerHost)
val actual = if (min < 0) 0 else min
runningSnapshot[it] = (runningSnapshot[it] ?: 0) + actual
actual
}
}
private fun getCandidates(): Map<Byte, List<ImageQueueElement>> {
return queueManager.pending().groupBy { it.host }
}
}
@@ -0,0 +1,217 @@
package me.vripper.services.download
import dev.failsafe.function.CheckedRunnable
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.entities.Status
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.DataAccessService
import me.vripper.services.HTTPService
import me.vripper.services.VGAuthService
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
import org.apache.hc.client5.http.cookie.Cookie
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.time.Instant
import java.util.*
import kotlin.io.path.Path
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.pathString
import kotlin.io.path.readLines
internal class ImageDownloadRunnable(
imageEntity: ImageEntity, settings: Settings
) : KoinComponent, CheckedRunnable {
class Context(val imageEntity: ImageEntity, val settings: Settings) {
private val log by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext = HttpClientContext.create().apply {
cookieStore = HTTPService.cookieStore
}
val requests = mutableListOf<HttpUriRequestBase>()
init {
ApplicationProperties.VRIPPER_DIR.listDirectoryEntries()
.filter { it.fileName.pathString.startsWith("cookies") }.forEach { cookiesPath ->
loadCookies(cookiesPath).also { cookies ->
cookies.forEach { cookie ->
if (HTTPService.cookieStore.cookies.find { it.name == cookie.name } == null) {
log.info("Applying cookie: ${cookie.name}")
HTTPService.cookieStore.addCookie(cookie)
} else {
log.warn("Cookie already loaded: ${cookie.name}")
}
}
}
}
}
private fun loadCookies(cookiesPath: Path): List<Cookie> {
return cookiesPath.readLines().map { it.trim() }.filter { it.isNotBlank() }.filter { !it.startsWith("#") }
.map { line ->
val cookieComponents = line.split("\t")
BasicClientCookie(cookieComponents[5], cookieComponents[6]).apply {
domain = cookieComponents[0]
isHttpOnly = cookieComponents[1].toBoolean()
path = cookieComponents[2]
isSecure = cookieComponents[3].toBoolean()
setExpiryDate(Instant.ofEpochSecond(cookieComponents[4].toLong()))
}
}.also {
log.info("Found ${it.size} cookies in $cookiesPath")
}
}
fun clear() {
runBlocking {
requests.forEach { it.abort() }
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
jobs.clear()
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
}
private val log by LoggerDelegate()
private val dataAccessService: DataAccessService by inject()
private val vgAuthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
val context = Context(imageEntity, settings)
var completed = false
var stopped = false
fun download() {
try {
context.imageEntity.status = Status.DOWNLOADING
context.imageEntity.downloaded = 0
dataAccessService.updateImage(context.imageEntity)
synchronized(context.imageEntity.postEntityId.toString().intern()) {
val post = dataAccessService.findPostByEntityId(context.imageEntity.postEntityId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataAccessService.updatePost(post)
vgAuthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${context.imageEntity.url} using ${context.imageEntity.host}")
val host = hosts.first { it.isSupported(context.imageEntity.url) }
val downloadedImage = host.downloadInternal(context)
log.debug("Resolved name for ${context.imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", context.imageEntity.url, downloadedImage.path)
synchronized(context.imageEntity.postEntityId.toString().intern()) {
val post = dataAccessService.findPostByEntityId(context.imageEntity.postEntityId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, context.imageEntity.index
)
if (context.imageEntity.downloaded == context.imageEntity.size && context.imageEntity.size > 0) {
context.imageEntity.status = Status.FINISHED
post.done += 1
post.downloaded += context.imageEntity.size
dataAccessService.updatePost(post)
} else {
context.imageEntity.status = Status.ERROR
}
dataAccessService.updateImage(context.imageEntity)
}
} catch (e: Exception) {
if (stopped) {
return
}
context.imageEntity.status = Status.ERROR
dataAccessService.updateImage(context.imageEntity)
throw DownloadException(e)
}
}
private fun checkImageTypeAndRename(
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
) {
val existingExtension = getExtension(downloadedImage.name).lowercase()
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
val extension = when (downloadedImage.type) {
ImageMimeType.IMAGE_BMP -> "BMP"
ImageMimeType.IMAGE_GIF -> "GIF"
ImageMimeType.IMAGE_JPEG -> "JPG"
ImageMimeType.IMAGE_PNG -> "PNG"
ImageMimeType.IMAGE_WEBP -> "WEBP"
}
val filename = if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
sanitize(
fileNameWithoutExtension
)
}.$extension"
try {
val downloadDestinationFolder = Path.of(downloadDirectory)
Files.createDirectories(downloadDestinationFolder)
val finalFilename = "${
if (context.settings.downloadSettings.forceOrder) String.format(
"%03d_", index + 1
) else ""
}$filename"
context.imageEntity.filename = finalFilename
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
} catch (e: Exception) {
throw HostException("Failed to rename the image", e)
} finally {
try {
Files.delete(downloadedImage.path)
} catch (_: IOException) {
}
}
}
override fun run() {
try {
if (stopped) {
return
}
download()
} finally {
completed = true
context.clear()
}
}
fun stop() {
stopped = true
context.clear()
dataAccessService.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
val that = other as ImageDownloadRunnable
return context.imageEntity.id == that.context.imageEntity.id
}
override fun hashCode(): Int {
return Objects.hash(context.imageEntity.id)
}
}
@@ -0,0 +1,3 @@
package me.vripper.services.download
internal data class ImageQueueElement(val imageEntityId: Long, val postEntityId: Long, val host: Byte)
@@ -0,0 +1,5 @@
package me.vripper.services.download
enum class MovePosition {
UP, DOWN, TOP, BOTTOM
}
@@ -0,0 +1,181 @@
package me.vripper.services.download
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import me.vripper.entities.Status
import me.vripper.event.ErrorCountEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.host.Host
import me.vripper.model.ErrorCount
import me.vripper.model.QueueState
import me.vripper.model.Rank
import me.vripper.services.DataAccessService
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.download.SharedLock.downloadManagerCondition
import me.vripper.services.download.SharedLock.downloadManagerLock
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.downloadRunner
import java.util.*
import kotlin.concurrent.withLock
internal class QueueManager(
private val dataAccessService: DataAccessService,
private val hosts: List<Host>,
private val settingsService: SettingsService,
private val retryPolicyService: RetryPolicyService,
private val eventBus: EventBus
) {
private val log by LoggerDelegate()
private val running: MutableList<ImageDownloadRunnable> = mutableListOf()
private val pending: MutableList<MutableList<ImageQueueElement>> = mutableListOf()
fun addPending(imageQueueElementList: List<ImageQueueElement>) {
pending.add(imageQueueElementList.toMutableList())
reportQueueState()
}
fun clearPending(postEntityId: Long? = null) {
val toProcess = if (postEntityId != null) pending.flatten()
.filter { it.postEntityId == postEntityId } else pending.flatten()
pending.forEach { it.removeAll(toProcess) }
pending.removeIf { it.isEmpty() }
reportQueueState()
}
fun runningTotalCount(): Map<Byte, Int> {
return hosts.associate { host ->
Pair(
host.hostId, running.count { it.context.imageEntity.host == host.hostId })
}
}
fun clearRunning(postEntityId: Long? = null) {
val toProcess =
if (postEntityId != null) running.filter { it.context.imageEntity.postEntityId == postEntityId } else running
toProcess.forEach { it.stop() }
while (toProcess.count { !it.completed } > 0) {
Thread.sleep(100)
}
running.removeAll(toProcess)
}
fun clearRunningRunnable(imageId: Long) {
running.removeIf { it.context.imageEntity.id == imageId }
}
fun accept(accepted: List<ImageQueueElement>) {
downloadManagerLock.withLock {
pending.forEach {
it.removeAll(accepted)
}
pending.removeIf { it.isEmpty() }
accepted.map {
ImageDownloadRunnable(
dataAccessService.findImageById(it.imageEntityId).orElseThrow(), settingsService.settings.copy()
)
}.forEach {
launch(it)
running.add(it)
}
reportQueueState()
}
}
fun pending(): List<ImageQueueElement> {
return pending.flatten()
}
fun isPending(postEntityId: Long): Boolean {
return pending.flatten().any { it.postEntityId == postEntityId }
}
fun isRunning(postEntityId: Long): Boolean {
return running.any { it.context.imageEntity.postEntityId == postEntityId }
}
fun move(postEntityId: Long, position: MovePosition) {
downloadManagerLock.withLock {
val index = pending.indexOfFirst { it.first().postEntityId == postEntityId }
when (position) {
MovePosition.UP -> {
val newIndex = if (index > 0) index - 1 else index
Collections.swap(pending, index, newIndex)
}
MovePosition.DOWN -> {
val newIndex = if (index < pending.size - 1) index + 1 else index
Collections.swap(pending, index, newIndex)
}
MovePosition.TOP -> {
if (index > 0) {
val element = pending.removeAt(index)
pending.addFirst(element)
}
}
MovePosition.BOTTOM -> {
if (index < pending.size - 1) {
val element = pending.removeAt(pending.size - 1)
pending.addLast(element)
}
}
}
reportQueueState()
}
}
fun getQueueState(): QueueState {
val ranks = downloadManagerLock.withLock {
pending.mapIndexed { index, elements -> Rank(elements.first().postEntityId, index + 1L) }
}
return QueueState(running.size, pending().size, ranks)
}
private fun launch(runnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${runnable.context.imageEntity.url}")
reportQueueState()
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${runnable.context.imageEntity.url}: "))
.with(downloadRunner).onFailure {
log.error(
"Failed to download ${runnable.context.imageEntity.url} after ${it.attemptCount} tries",
it.exception
)
val image = runnable.context.imageEntity
image.status = Status.ERROR
dataAccessService.updateImage(image)
}.onComplete {
afterJobFinish(runnable)
log.debug(
"Finished downloading ${runnable.context.imageEntity.url}"
)
}.runAsync(runnable)
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
downloadManagerLock.withLock {
val image = imageDownloadRunnable.context.imageEntity
clearRunningRunnable(imageDownloadRunnable.context.imageEntity.id)
if (!isPending(image.postEntityId) && !isRunning(image.postEntityId) && !imageDownloadRunnable.stopped) {
dataAccessService.finishPost(image.postEntityId, true)
}
reportQueueState()
log.debug("[{}] Event published: ErrorCountEvent after job finish", System.currentTimeMillis())
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataAccessService.countImagesInError())))
downloadManagerCondition.signal()
}
}
private fun reportQueueState() {
val queueState = getQueueState()
log.debug(
"[{}] Publishing event: QueueStateEvent(running={}, remaining={})",
System.currentTimeMillis(),
queueState.running,
queueState.remaining
)
eventBus.publishEvent(QueueStateEvent(queueState))
}
}
@@ -0,0 +1,9 @@
package me.vripper.services.download
import java.util.concurrent.locks.Condition
import java.util.concurrent.locks.ReentrantLock
object SharedLock {
val downloadManagerLock = ReentrantLock()
val downloadManagerCondition: Condition = downloadManagerLock.newCondition()
}
@@ -1,11 +0,0 @@
package me.vripper.tables
import org.jetbrains.exposed.dao.id.LongIdTable
import org.jetbrains.exposed.sql.javatime.datetime
object LogTable : LongIdTable(name = "LOG", columnName = "ID") {
val type = varchar("TYPE", 8)
val status = varchar("STATUS", 10)
val time = datetime("TIME")
val message = varchar("MESSAGE", 500)
}
@@ -1,8 +0,0 @@
package me.vripper.tables
import org.jetbrains.exposed.sql.Table
object MetadataTable : Table(name = "METADATA") {
val postId = long("POST_ID")
val data = varchar("DATA", 1_000_000)
}
@@ -1,112 +0,0 @@
package me.vripper.tasks
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.download.DownloadService
import me.vripper.entities.LogEntryEntity
import me.vripper.model.ThreadPostId
import me.vripper.parser.PostItem
import me.vripper.parser.PostLookupAPIParser
import me.vripper.services.DataTransaction
import me.vripper.services.MetadataService
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.Tasks
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
class AddPostRunnable(private val items: List<ThreadPostId>) : KoinComponent, Runnable {
private val log by me.vripper.delegate.LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val settingsService: SettingsService by inject()
private val downloadService: DownloadService by inject()
private val threadCacheService: ThreadCacheService by inject()
private val metadataService: MetadataService by inject()
override fun run() {
try {
Tasks.increment()
val toProcess = mutableListOf<PostItem>()
for ((threadId, postId) in items) {
if (dataTransaction.exists(postId)) {
continue
}
val link =
"https://${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
val cachedThread = threadCacheService.getIfPresent(threadId)
val threadItem = cachedThread ?: runBlocking {
RequestLimit.semaphore.withPermit {
PostLookupAPIParser(
threadId, postId
).parse()
}
}
if (threadItem == null) {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.POST,
status = LogEntryEntity.Status.ERROR,
message = "Failed to load $link"
)
)
continue
} else {
if (threadItem.error.isNotBlank()) {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.POST,
status = LogEntryEntity.Status.ERROR,
message = "Error loading $link: ${threadItem.error}"
)
)
continue
} else if (threadItem.postItemList.isEmpty()) {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.POST,
status = LogEntryEntity.Status.ERROR,
message = "Nothing found for $link"
)
)
continue
}
}
val postItem = threadItem.postItemList.firstOrNull { it.postId == postId }
if (postItem == null) {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.POST,
status = LogEntryEntity.Status.ERROR,
message = "Unable to load $link"
)
)
continue
}
toProcess.add(postItem)
}
val posts = try {
dataTransaction.newPosts(toProcess.toList())
} catch (e: Exception) {
e.printStackTrace()
emptyList()
}
posts.forEach {
metadataService.fetchMetadata(it.postId)
}
if (settingsService.settings.downloadSettings.autoStart) {
downloadService.restartAll(posts)
}
} catch (e: Exception) {
val error = String.format("Error when adding galleries")
log.error(error, e)
} finally {
Tasks.decrement()
}
}
}
@@ -0,0 +1,87 @@
package me.vripper.tasks
import me.vripper.model.ThreadPostId
import me.vripper.services.DataAccessService
import me.vripper.services.MetadataService
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.services.download.DownloadService
import me.vripper.utilities.LoggerDelegate
import me.vripper.vgapi.PostItem
import me.vripper.vgapi.PostLookupAPIParser
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponent, Runnable {
private val log by LoggerDelegate()
private val dataAccessService: DataAccessService by inject()
private val settingsService: SettingsService by inject()
private val downloadService: DownloadService by inject()
private val threadCacheService: ThreadCacheService by inject()
private val metadataService: MetadataService by inject()
override fun run() {
try {
Tasks.increment()
val toProcess = mutableListOf<PostItem>()
for ((threadId, postId) in items) {
if (dataAccessService.existsPostId(postId)) {
log.info("Post $postId already loaded")
continue
}
val link =
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
val cachedThread = threadCacheService.getIfPresent(threadId)
val threadItem = cachedThread ?:
PostLookupAPIParser(
threadId, postId
).parse()
if (threadItem == null) {
log.error("Failed to load $link")
continue
} else {
if (threadItem.error.isNotBlank()) {
log.error("Error loading $link: ${threadItem.error}")
continue
} else if (threadItem.postItemList.isEmpty()) {
log.error("Nothing found for $link")
continue
}
}
val postItem = threadItem.postItemList.firstOrNull { it.postId == postId }
if (postItem == null) {
log.error("Unable to load $link")
continue
}
toProcess.add(postItem)
}
if (toProcess.isEmpty()) {
return
}
val posts = try {
dataAccessService.newPosts(toProcess.toList())
} catch (e: Exception) {
e.printStackTrace()
emptyList()
}
posts.forEach {
metadataService.fetchMetadata(it)
}
if (settingsService.settings.downloadSettings.autoStart) {
downloadService.restartAll(posts)
}
} catch (e: Exception) {
val error = String.format("Error when adding galleries")
log.error(error, e)
} finally {
Tasks.decrement()
}
}
}
@@ -0,0 +1,111 @@
package me.vripper.tasks
import me.vripper.entities.MetadataEntity
import me.vripper.entities.PostEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.services.DataAccessService
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.w3c.dom.Node
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
internal class FetchMetadataTask(
private val postEntity: PostEntity,
) : KoinComponent, Runnable {
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
private val log by LoggerDelegate()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
private val httpService: HTTPService by inject()
private val dataAccessService: DataAccessService by inject()
override fun run() {
try {
if (!settingsService.settings.viperSettings.fetchMetadata) {
log.debug("Fetching metadata is disabled")
return
}
Tasks.increment()
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postEntity.vgPostId.toString()
)
}.build()).also { it.setAbsoluteRequestUri(true) }
RequestLimit.getPermit(1)
log.info("{}", httpGet)
val response = httpService.client.execute(httpGet, vgAuthService.createVgContext()) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
EntityUtils.toString(it.entity)
}
val document = HtmlUtils.clean(response)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_${postEntity.vgPostId}']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'${postEntity.vgPostId}'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'${postEntity.vgPostId}'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postEntity.vgPostId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
val metadataEntity = MetadataEntity(postEntity.id, MetadataEntity.Data(postedBy, titles))
dataAccessService.saveMetadata(metadataEntity)
} finally {
Tasks.decrement()
}
}
private fun findTitleInContent(node: Node): List<String> {
val altTitle: MutableList<String> = mutableListOf()
findTitle(node, altTitle, AtomicBoolean(true))
return altTitle.stream().distinct().collect(Collectors.toList())
}
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
if (!keepGoing.get()) {
return
}
if (node.nodeName == "a" || node.nodeName == "img") {
keepGoing.set(false)
return
}
if (node.nodeType == Node.ELEMENT_NODE) {
for (i in 0 until node.childNodes.length) {
val item = node.childNodes.item(i)
findTitle(item, altTitle, keepGoing)
if (!keepGoing.get()) {
return
}
}
} else if (node.nodeType == Node.TEXT_NODE) {
val text = node.textContent.trim()
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
text.lowercase().contains(e.lowercase())
}) {
altTitle.add(text)
}
}
}
}
@@ -1,71 +0,0 @@
package me.vripper.tasks
import me.vripper.entities.LogEntryEntity
import me.vripper.entities.LogEntryEntity.Status.ERROR
import me.vripper.entities.PostEntity
import me.vripper.services.DataTransaction
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.utilities.Tasks
import me.vripper.utilities.formatToString
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.message.BasicNameValuePair
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
class LeaveThanksRunnable(
private val postEntity: PostEntity,
private val authenticated: Boolean,
private val context: HttpClientContext
) : KoinComponent, Runnable {
private val log by me.vripper.delegate.LoggerDelegate()
private val cm: HTTPService by inject()
private val settingsService: SettingsService by inject()
private val dataTransaction: DataTransaction by inject()
override fun run() {
try {
Tasks.increment()
if (!settingsService.settings.viperSettings.login || !authenticated) {
return
}
if (!settingsService.settings.viperSettings.thanks) {
return
}
val postThanks =
HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.postId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
)
)
it.addHeader("Referer", settingsService.settings.viperSettings.host)
it.addHeader(
"Host",
settingsService.settings.viperSettings.host.replace("https://", "")
.replace("http://", "")
)
}
cm.client.execute(postThanks, context) { }
} catch (e: Exception) {
val error = "Failed to leave a thanks for $postEntity"
log.error(error, e)
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.THANKS, status = ERROR, message = """
$error
${e.formatToString()}
""".trimIndent()
)
)
} finally {
Tasks.decrement()
}
}
}
@@ -0,0 +1,58 @@
package me.vripper.tasks
import me.vripper.entities.PostEntity
import me.vripper.exception.VripperException
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.message.BasicNameValuePair
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
internal class LeaveThanksTask(
private val postEntity: PostEntity,
private val authenticated: Boolean,
private val context: HttpClientContext
) : KoinComponent, Runnable {
private val log by LoggerDelegate()
private val cm: HTTPService by inject()
private val settingsService: SettingsService by inject()
override fun run() {
try {
Tasks.increment()
if (!settingsService.settings.viperSettings.login || !authenticated) {
return
}
if (!settingsService.settings.viperSettings.thanks) {
return
}
val postThanks = HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.vgPostId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
)
)
it.setAbsoluteRequestUri(true)
}
RequestLimit.getPermit(1)
log.info("Posting {}", postThanks)
cm.client.execute(postThanks, context) { response ->
if (response.code / 100 != 2) {
throw VripperException("Unexpected response code '${response.code}' for $postThanks")
}
}
} catch (e: Exception) {
log.error("Failed to leave a thanks for $postEntity", e)
} finally {
Tasks.decrement()
}
}
}
@@ -0,0 +1,28 @@
package me.vripper.tasks
import me.vripper.event.EventBus
import me.vripper.event.LoadingTasks
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
internal object Tasks : KoinComponent {
private val eventBus: EventBus by inject()
private var current = 0
@Synchronized
fun increment() {
if (current == 0) {
eventBus.publishEvent(LoadingTasks(true))
}
current += 1
}
@Synchronized
fun decrement() {
current -= 1
if (current == 0) {
eventBus.publishEvent(LoadingTasks(false))
}
}
}
@@ -1,87 +0,0 @@
package me.vripper.tasks
import me.vripper.entities.LogEntryEntity
import me.vripper.entities.LogEntryEntity.Status.ERROR
import me.vripper.entities.ThreadEntity
import me.vripper.model.Settings
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.GLOBAL_EXECUTOR
import me.vripper.utilities.Tasks
import me.vripper.utilities.formatToString
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.util.concurrent.CompletableFuture
class ThreadLookupRunnable(private val threadId: Long, private val settings: Settings) : KoinComponent, Runnable {
private val log by me.vripper.delegate.LoggerDelegate()
private val dataTransaction by inject<DataTransaction>()
private val settingsService by inject<SettingsService>()
private val threadCacheService by inject<ThreadCacheService>()
private val link: String = "${settingsService.settings.viperSettings.host}/threads/$threadId"
override fun run() {
try {
Tasks.increment()
if (dataTransaction.findThreadByThreadId(threadId).isEmpty) {
val threadLookupResult = threadCacheService[threadId]
if (threadLookupResult.error.isNotBlank()) {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.THREAD,
status = LogEntryEntity.Status.ERROR,
message = "Error loading $link: ${threadLookupResult.error}"
)
)
return
}
if (threadLookupResult.postItemList.isEmpty()) {
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.THREAD,
status = LogEntryEntity.Status.ERROR,
message = "Nothing found for $link"
)
)
return
}
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
CompletableFuture.runAsync(AddPostRunnable(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
)
}), GLOBAL_EXECUTOR)
} else {
try {
dataTransaction.save(
ThreadEntity(
title = threadLookupResult.title,
link = link,
threadId = threadId,
total = threadLookupResult.postItemList.size
)
)
} catch (e: Exception) {
e.printStackTrace()
}
}
}
} catch (e: Exception) {
val error = "Error when processing $link"
log.error(error, e)
dataTransaction.saveLog(
LogEntryEntity(
type = LogEntryEntity.Type.THREAD, status = ERROR, message = """
$error
${e.formatToString()}
""".trimIndent()
)
)
} finally {
Tasks.decrement()
}
}
}
@@ -0,0 +1,60 @@
package me.vripper.tasks
import me.vripper.entities.ThreadEntity
import me.vripper.model.Settings
import me.vripper.model.ThreadPostId
import me.vripper.services.DataAccessService
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.taskRunner
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
internal class ThreadLookupTask(private val threadId: Long, private val settings: Settings) : KoinComponent, Runnable {
private val log by LoggerDelegate()
private val dataAccessService by inject<DataAccessService>()
private val settingsService by inject<SettingsService>()
private val threadCacheService by inject<ThreadCacheService>()
private val link: String = "${settingsService.settings.viperSettings.host}/threads/$threadId"
override fun run() {
try {
Tasks.increment()
if (dataAccessService.findThreadByThreadId(threadId).isEmpty) {
val threadLookupResult = threadCacheService[threadId]
if (threadLookupResult.error.isNotBlank()) {
log.error("Error loading $link: ${threadLookupResult.error}")
return
}
if (threadLookupResult.postItemList.isEmpty()) {
log.error("Nothing found for $link")
return
}
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
taskRunner.submit(
AddPostTask(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
)
})
)
} else {
dataAccessService.save(
ThreadEntity(
title = threadLookupResult.title,
link = link,
threadId = threadId,
total = threadLookupResult.postItemList.size
)
)
}
}
} catch (e: Exception) {
log.error("Error when processing $link", e)
} finally {
Tasks.decrement()
}
}
}

Some files were not shown because too many files have changed in this diff Show More