mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2220410f02 |
@@ -8,12 +8,12 @@ jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ]
|
||||
os: [ ubuntu-latest, windows-latest, macos-latest ]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up jdk21
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: '21'
|
||||
@@ -21,29 +21,37 @@ jobs:
|
||||
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
|
||||
settings-path: ${{ github.workspace }} # location for the settings.xml file
|
||||
|
||||
- name: Build gui jar
|
||||
run: |
|
||||
mvn -N -q install --file pom.xml
|
||||
mvn -B -q install --file vripper-core/pom.xml
|
||||
mvn -B -q install --file vripper-gui/pom.xml
|
||||
mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
|
||||
- name: Build parent project
|
||||
run: mvn -N -q install --file pom.xml
|
||||
|
||||
- name: Build Core Jar
|
||||
run: mvn -B -q install --file vripper-core/pom.xml
|
||||
|
||||
- name: Build GUI Jar
|
||||
run: mvn -B -q install --file vripper-gui/pom.xml
|
||||
|
||||
- name: Rename GUI Jar
|
||||
run: mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
|
||||
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Release gui Jar
|
||||
name: Release GUI Jar
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
|
||||
|
||||
# Start building WEB jar in ubuntu only
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Build web Jar
|
||||
name: Build WEB Jar
|
||||
run: |
|
||||
mvn -B -q install --file vripper-web-ui/pom.xml
|
||||
mvn -B -q install --file vripper-web/pom.xml
|
||||
mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
|
||||
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Release web Jar
|
||||
name: Rename WEB Jar
|
||||
run: mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
|
||||
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Release WEB Jar
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
|
||||
@@ -54,7 +62,7 @@ jobs:
|
||||
cp vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar jpackage/jar/vripper-gui.jar
|
||||
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Package for Linux
|
||||
name: Package Linux
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.png
|
||||
@@ -64,7 +72,7 @@ jobs:
|
||||
mv dist/vripper_${{ github.event.release.tag_name }}-1_amd64.deb dist/vripper-linux-${{ github.event.release.tag_name }}_amd64.deb
|
||||
|
||||
- if: matrix.os == 'windows-latest'
|
||||
name: Package for Windows
|
||||
name: Package Windows
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.ico
|
||||
@@ -75,24 +83,14 @@ jobs:
|
||||
ren VRipper-${{ github.event.release.tag_name }}.exe vripper-windows-installer-${{ github.event.release.tag_name }}.exe
|
||||
|
||||
- if: matrix.os == 'macos-latest'
|
||||
name: Package for macOS(arm64)
|
||||
name: Package macOS
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
|
||||
|
||||
- if: matrix.os == 'macos-13'
|
||||
name: Package for macOS(x86_64)
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
|
||||
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Zip Ubuntu portable
|
||||
@@ -113,22 +111,13 @@ jobs:
|
||||
filename: 'vripper-windows-portable-${{ github.event.release.tag_name }}.zip'
|
||||
|
||||
- if: matrix.os == 'macos-latest'
|
||||
name: Zip macOS(arm64) portable
|
||||
name: Zip MacOS portable
|
||||
uses: thedoctor0/zip-release@0.7.1
|
||||
with:
|
||||
type: 'zip'
|
||||
directory: 'jpackage/dist'
|
||||
path: 'VRipper.app'
|
||||
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip'
|
||||
|
||||
- if: matrix.os == 'macos-13'
|
||||
name: Zip macOS(x86_64) portable
|
||||
uses: thedoctor0/zip-release@0.7.1
|
||||
with:
|
||||
type: 'zip'
|
||||
directory: 'jpackage/dist'
|
||||
path: 'VRipper.app'
|
||||
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip'
|
||||
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.zip'
|
||||
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Release packages for Linux
|
||||
@@ -149,19 +138,10 @@ jobs:
|
||||
jpackage/dist/vripper-windows-portable-${{ github.event.release.tag_name }}.zip
|
||||
|
||||
- if: matrix.os == 'macos-latest'
|
||||
name: Release packages for macOS(arm64)
|
||||
name: Release packages for macOS
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: |
|
||||
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
|
||||
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
|
||||
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip
|
||||
|
||||
- if: matrix.os == 'macos-13'
|
||||
name: Release packages for macOS(x86_64)
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: |
|
||||
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
|
||||
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
|
||||
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip
|
||||
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
|
||||
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
|
||||
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.zip
|
||||
|
||||
@@ -1,28 +1,19 @@
|
||||
# VRipper!
|
||||
|
||||
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to)
|
||||
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to).
|
||||
|
||||
|
||||

|
||||
|
||||
## Donation
|
||||
To support this project, you can make a donation to its current maintainer
|
||||
|
||||
[](https://buymeacoffee.com/devclaw)
|
||||
|
||||
Or with Cryptocurrency
|
||||
|
||||
ETH: 0xDdac82B16dC5E3D742fc915ffF583D8548A301cA
|
||||
|
||||
BTC: bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr
|
||||
|
||||
## Requirements
|
||||
Direct access to `vipergirls.to` domain.
|
||||
|
||||
## Installing VRipper
|
||||
|
||||
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
|
||||
:---:|:---:|:---:|:---:|:---:
|
||||
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-windows-installer-6.5.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-windows-installer-6.5.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-windows-portable-6.5.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-portable-6.5.0.x86_64.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.arm64.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-portable-6.5.0.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-linux-6.5.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-linux-6.5.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-linux-portable-6.5.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-noarch-gui-6.5.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-noarch-web-6.5.0.jar)
|
||||
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
|
||||
:---:|:---:|:---:|:---:
|
||||
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-installer-6.0.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-installer-6.0.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-portable-6.0.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-6.0.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-6.0.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-portable-6.0.0.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-6.0.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-6.0.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-portable-6.0.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-noarch-gui-6.0.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-noarch-web-6.0.0.jar)
|
||||
|
||||
Source code and previous versions are available on the [Releases page](https://github.com/death-claw/vripper-project/releases).
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<revision>6.5.0</revision>
|
||||
<revision>6.0.0</revision>
|
||||
<kotlin.version>2.0.21</kotlin.version>
|
||||
<slf4j.version>2.0.16</slf4j.version>
|
||||
<logback.version>1.5.12</logback.version>
|
||||
@@ -25,7 +25,7 @@
|
||||
<koin.version>4.0.0</koin.version>
|
||||
<h2.version>2.2.224</h2.version>
|
||||
<htmlcleaner.version>2.29</htmlcleaner.version>
|
||||
<failsafe.version>3.3.2</failsafe.version>
|
||||
<failsafe.version>2.4.4</failsafe.version>
|
||||
<caffeine.version>3.1.8</caffeine.version>
|
||||
<jna-platform.version>5.14.0</jna-platform.version>
|
||||
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
|
||||
@@ -142,8 +142,8 @@
|
||||
<version>${htmlcleaner.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.failsafe</groupId>
|
||||
<artifactId>failsafe</artifactId>
|
||||
<groupId>net.jodah</groupId>
|
||||
<version>${failsafe.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
||||
@@ -69,7 +69,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>failsafe</artifactId>
|
||||
<groupId>dev.failsafe</groupId>
|
||||
<groupId>net.jodah</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>caffeine</artifactId>
|
||||
@@ -108,11 +108,6 @@
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-kotlin</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.bucket4j</groupId>
|
||||
<artifactId>bucket4j_jdk17-core</artifactId>
|
||||
<version>8.14.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
@@ -141,14 +136,14 @@
|
||||
<executions>
|
||||
<execution>
|
||||
<id>compile</id>
|
||||
<phase>process-sources</phase>
|
||||
<phase>compile</phase>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<jvmTarget>21</jvmTarget>
|
||||
<jvmTarget>1.8</jvmTarget>
|
||||
<compilerPlugins>
|
||||
<plugin>kotlinx-serialization</plugin>
|
||||
</compilerPlugins>
|
||||
|
||||
@@ -2,11 +2,11 @@ package me.vripper
|
||||
|
||||
import me.vripper.data.repositories.ImageRepository
|
||||
import me.vripper.data.repositories.MetadataRepository
|
||||
import me.vripper.data.repositories.PostRepository
|
||||
import me.vripper.data.repositories.PostDownloadStateRepository
|
||||
import me.vripper.data.repositories.ThreadRepository
|
||||
import me.vripper.data.repositories.impl.ImageRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.PostRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.PostDownloadStateRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
|
||||
import me.vripper.download.DownloadService
|
||||
import me.vripper.event.EventBus
|
||||
@@ -29,8 +29,8 @@ val coreModule = module {
|
||||
single<ImageRepository> {
|
||||
ImageRepositoryImpl()
|
||||
}
|
||||
single<PostRepository> {
|
||||
PostRepositoryImpl()
|
||||
single<PostDownloadStateRepository> {
|
||||
PostDownloadStateRepositoryImpl()
|
||||
}
|
||||
single<MetadataRepository> {
|
||||
MetadataRepositoryImpl()
|
||||
@@ -69,7 +69,7 @@ val coreModule = module {
|
||||
} bind IAppEndpointService::class
|
||||
|
||||
single<MetadataService> {
|
||||
MetadataService(get(), get())
|
||||
MetadataService(get(), get(), get(), get())
|
||||
}
|
||||
single {
|
||||
AcidimgHost(get(), get(), get())
|
||||
|
||||
+4
-3
@@ -1,11 +1,12 @@
|
||||
package me.vripper.data.repositories
|
||||
|
||||
import me.vripper.entities.PostEntity
|
||||
import java.util.*
|
||||
|
||||
internal interface PostRepository {
|
||||
internal interface PostDownloadStateRepository {
|
||||
fun save(postEntities: List<PostEntity>): List<PostEntity>
|
||||
fun findByPostId(postId: Long): PostEntity?
|
||||
fun findById(id: Long): PostEntity?
|
||||
fun findByPostId(postId: Long): Optional<PostEntity>
|
||||
fun findById(id: Long): Optional<PostEntity>
|
||||
fun findCompleted(): List<Long>
|
||||
fun findAll(): List<PostEntity>
|
||||
fun existByPostId(postId: Long): Boolean
|
||||
+17
-7
@@ -1,6 +1,6 @@
|
||||
package me.vripper.data.repositories.impl
|
||||
|
||||
import me.vripper.data.repositories.PostRepository
|
||||
import me.vripper.data.repositories.PostDownloadStateRepository
|
||||
import me.vripper.data.tables.PostTable
|
||||
import me.vripper.entities.PostEntity
|
||||
import me.vripper.entities.Status
|
||||
@@ -8,9 +8,10 @@ import org.jetbrains.exposed.sql.*
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
|
||||
import org.jetbrains.exposed.sql.transactions.TransactionManager
|
||||
import java.sql.Connection
|
||||
import java.util.*
|
||||
|
||||
internal class PostRepositoryImpl :
|
||||
PostRepository {
|
||||
internal class PostDownloadStateRepositoryImpl :
|
||||
PostDownloadStateRepository {
|
||||
|
||||
private val delimiter = ";"
|
||||
|
||||
@@ -36,11 +37,15 @@ internal class PostRepositoryImpl :
|
||||
}.map(::transform)
|
||||
}
|
||||
|
||||
override fun findByPostId(postId: Long): PostEntity? {
|
||||
override fun findByPostId(postId: Long): Optional<PostEntity> {
|
||||
val result = PostTable.selectAll().where {
|
||||
PostTable.postId eq postId
|
||||
}.map(::transform)
|
||||
return result.firstOrNull()
|
||||
return if (result.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(result.first())
|
||||
}
|
||||
}
|
||||
|
||||
override fun findCompleted(): List<Long> {
|
||||
@@ -49,11 +54,16 @@ internal class PostRepositoryImpl :
|
||||
}.map { it[PostTable.postId] }
|
||||
}
|
||||
|
||||
override fun findById(id: Long): PostEntity? {
|
||||
override fun findById(id: Long): Optional<PostEntity> {
|
||||
val result = PostTable.selectAll().where {
|
||||
PostTable.id eq id
|
||||
}.map { transform(it) }
|
||||
return result.firstOrNull()
|
||||
|
||||
return if (result.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(result.first())
|
||||
}
|
||||
}
|
||||
|
||||
override fun findAll(): List<PostEntity> {
|
||||
@@ -1,8 +1,9 @@
|
||||
package me.vripper.download
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.RetryPolicy
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Runnable
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.entities.PostEntity
|
||||
import me.vripper.entities.Status
|
||||
@@ -17,8 +18,10 @@ import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.downloadRunner
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.RetryPolicy
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
@@ -36,6 +39,7 @@ internal class DownloadService(
|
||||
private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
|
||||
private val lock = ReentrantLock()
|
||||
private val condition = lock.newCondition()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
|
||||
init {
|
||||
Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
|
||||
@@ -45,14 +49,14 @@ internal class DownloadService(
|
||||
lock.withLock {
|
||||
candidates.addAll(getCandidates(candidateCount()))
|
||||
candidates.forEach {
|
||||
if (canRun(it.imageEntity.host)) {
|
||||
if (canRun(it.context.imageEntity.host)) {
|
||||
accepted.add(it)
|
||||
running[it.imageEntity.host]!!.add(it)
|
||||
log.debug("${it.imageEntity.url} accepted to run")
|
||||
running[it.context.imageEntity.host]!!.add(it)
|
||||
log.debug("${it.context.imageEntity.url} accepted to run")
|
||||
}
|
||||
}
|
||||
accepted.forEach {
|
||||
pending[it.imageEntity.host]?.remove(it)
|
||||
pending[it.context.imageEntity.host]?.remove(it)
|
||||
scheduleForDownload(it)
|
||||
}
|
||||
accepted.clear()
|
||||
@@ -70,10 +74,14 @@ internal class DownloadService(
|
||||
fun stop(postIds: List<Long> = emptyList()) {
|
||||
if (postIds.isNotEmpty()) {
|
||||
stopInternal(postIds)
|
||||
eventBus.publishEvent(StoppedEvent(postIds))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(StoppedEvent(postIds))
|
||||
}
|
||||
} else {
|
||||
stopAll()
|
||||
eventBus.publishEvent(StoppedEvent(listOf(-1)))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(StoppedEvent(listOf(-1)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,9 +89,8 @@ internal class DownloadService(
|
||||
if (postEntityIds.isNotEmpty()) {
|
||||
restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
|
||||
} else {
|
||||
restart(
|
||||
dataTransaction.findAllPosts()
|
||||
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
|
||||
restart(dataTransaction.findAllPosts()
|
||||
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,6 +127,10 @@ internal class DownloadService(
|
||||
)
|
||||
}
|
||||
|
||||
toProcess.keys.forEach {
|
||||
vgAuthService.leaveThanks(it)
|
||||
}
|
||||
|
||||
|
||||
toProcess.entries.forEach { (post, images) ->
|
||||
images.forEach { image ->
|
||||
@@ -139,18 +150,18 @@ internal class DownloadService(
|
||||
}
|
||||
|
||||
private fun isPending(postId: Long): Boolean {
|
||||
return pending.values.flatten().any { it.imageEntity.postId == postId }
|
||||
return pending.values.flatten().any { it.context.imageEntity.postId == postId }
|
||||
}
|
||||
|
||||
private fun isRunning(postId: Long): Boolean {
|
||||
return running.values.flatten().any { it.imageEntity.postId == postId }
|
||||
return running.values.flatten().any { it.context.imageEntity.postId == postId }
|
||||
}
|
||||
|
||||
private fun stopAll() {
|
||||
lock.withLock {
|
||||
pending.values.clear()
|
||||
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
|
||||
while (running.values.flatten().count { !it.completed } > 0) {
|
||||
while (running.values.flatten().count { !it.context.completed } > 0) {
|
||||
Thread.sleep(100)
|
||||
}
|
||||
dataTransaction.findAllNonCompletedPostIds().forEach {
|
||||
@@ -164,13 +175,13 @@ internal class DownloadService(
|
||||
lock.withLock {
|
||||
for (postId in postIds) {
|
||||
pending.values.forEach { pending ->
|
||||
pending.removeIf { it.imageEntity.postId == postId }
|
||||
pending.removeIf { it.context.imageEntity.postId == postId }
|
||||
}
|
||||
running.values.flatten()
|
||||
.filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
|
||||
.filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
|
||||
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
|
||||
while (running.values.flatten()
|
||||
.count { !it.completed && it.imageEntity.postId == postId } > 0
|
||||
.count { !it.context.completed && it.context.imageEntity.postId == postId } > 0
|
||||
) {
|
||||
Thread.sleep(100)
|
||||
}
|
||||
@@ -208,7 +219,7 @@ internal class DownloadService(
|
||||
|
||||
val list: List<ImageDownloadRunnable> =
|
||||
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
|
||||
.thenComparingInt { it.imageEntity.index })
|
||||
.thenComparingInt { it.context.imageEntity.index })
|
||||
|
||||
for (imageDownloadRunnable in list) {
|
||||
val count = hostIntegerMap[host] ?: 0
|
||||
@@ -224,42 +235,53 @@ internal class DownloadService(
|
||||
}
|
||||
|
||||
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
|
||||
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
|
||||
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
|
||||
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
|
||||
.with(downloadRunner)
|
||||
.onFailure {
|
||||
log.error(
|
||||
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
|
||||
it.exception
|
||||
)
|
||||
val image = imageDownloadRunnable.imageEntity
|
||||
image.status = Status.ERROR
|
||||
dataTransaction.updateImage(image)
|
||||
log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
|
||||
GlobalScopeCoroutine.launch {
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
|
||||
}
|
||||
.onComplete {
|
||||
afterJobFinish(imageDownloadRunnable)
|
||||
eventBus.publishEvent(
|
||||
QueueStateEvent(
|
||||
QueueState(
|
||||
runningCount(), pendingCount()
|
||||
try {
|
||||
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload("Failed to download ${imageDownloadRunnable.context.imageEntity.url}: "))
|
||||
.onFailure {
|
||||
log.error(
|
||||
"Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
|
||||
it.failure
|
||||
)
|
||||
)
|
||||
)
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
|
||||
log.debug(
|
||||
"Finished downloading ${imageDownloadRunnable.imageEntity.url}"
|
||||
)
|
||||
}.runAsync(imageDownloadRunnable)
|
||||
val image = imageDownloadRunnable.context.imageEntity
|
||||
image.status = Status.ERROR
|
||||
dataTransaction.updateImage(image)
|
||||
}
|
||||
.onComplete {
|
||||
afterJobFinish(imageDownloadRunnable)
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(
|
||||
QueueStateEvent(
|
||||
QueueState(
|
||||
runningCount(), pendingCount()
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
|
||||
}
|
||||
log.debug(
|
||||
"Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
|
||||
)
|
||||
}.run(imageDownloadRunnable::run)
|
||||
} catch (e: Exception) {
|
||||
log.error("Download Failure", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
|
||||
lock.withLock {
|
||||
val image = imageDownloadRunnable.imageEntity
|
||||
val image = imageDownloadRunnable.context.imageEntity
|
||||
running[image.host]!!.remove(imageDownloadRunnable)
|
||||
if (!isPending(image.postId) && !isRunning(
|
||||
image.postId
|
||||
) && !imageDownloadRunnable.stopped
|
||||
) && !imageDownloadRunnable.context.stopped
|
||||
) {
|
||||
dataTransaction.finishPost(image.postId, true)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package me.vripper.download
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.model.Settings
|
||||
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
|
||||
@@ -9,21 +8,10 @@ import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.koin.core.component.KoinComponent
|
||||
|
||||
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val jobs = mutableListOf<Job>()
|
||||
val httpContext: HttpClientContext =
|
||||
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
|
||||
val requests = mutableListOf<HttpUriRequestBase>()
|
||||
val postId = imageEntity.postIdRef
|
||||
|
||||
fun cancelCoroutines() {
|
||||
runBlocking {
|
||||
coroutineScope.cancel()
|
||||
jobs.forEach { job -> job.cancelAndJoin() }
|
||||
}
|
||||
}
|
||||
|
||||
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
|
||||
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
|
||||
}
|
||||
var stopped = false
|
||||
var completed = false
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
package me.vripper.download
|
||||
|
||||
import dev.failsafe.function.CheckedRunnable
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.entities.Status
|
||||
import me.vripper.exception.DownloadException
|
||||
@@ -9,12 +8,12 @@ import me.vripper.host.DownloadedImage
|
||||
import me.vripper.host.Host
|
||||
import me.vripper.host.ImageMimeType
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.services.*
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils.getExtension
|
||||
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
|
||||
import me.vripper.utilities.PathUtils.sanitize
|
||||
import net.jodah.failsafe.function.CheckedRunnable
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.io.IOException
|
||||
@@ -26,28 +25,26 @@ import kotlin.io.path.Path
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
internal class ImageDownloadRunnable(
|
||||
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
|
||||
private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
|
||||
) : KoinComponent, CheckedRunnable {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
private val vgauthService: VGAuthService by inject()
|
||||
private val hosts: List<Host> = getKoin().getAll()
|
||||
var completed = false
|
||||
var stopped = false
|
||||
|
||||
private lateinit var context: ImageDownloadContext
|
||||
val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
|
||||
|
||||
@Throws(DownloadException::class)
|
||||
fun download() {
|
||||
try {
|
||||
imageEntity.status = Status.DOWNLOADING
|
||||
imageEntity.downloaded = 0
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val post = dataTransaction.findPostById(context.postId).orElseThrow()
|
||||
if (post.status != Status.DOWNLOADING) {
|
||||
post.status = Status.DOWNLOADING
|
||||
dataTransaction.updatePost(post)
|
||||
vgauthService.leaveThanks(post)
|
||||
}
|
||||
}
|
||||
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
|
||||
@@ -56,7 +53,7 @@ internal class ImageDownloadRunnable(
|
||||
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
|
||||
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val post = dataTransaction.findPostById(context.postId).orElseThrow()
|
||||
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
|
||||
checkImageTypeAndRename(
|
||||
downloadDirectory, downloadedImage, imageEntity.index
|
||||
@@ -72,7 +69,7 @@ internal class ImageDownloadRunnable(
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (stopped) {
|
||||
if (context.stopped) {
|
||||
return
|
||||
}
|
||||
imageEntity.status = Status.ERROR
|
||||
@@ -121,26 +118,18 @@ internal class ImageDownloadRunnable(
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(Exception::class)
|
||||
override fun run() {
|
||||
context = ImageDownloadContext(imageEntity, settings)
|
||||
try {
|
||||
if (stopped) {
|
||||
if (context.stopped) {
|
||||
return
|
||||
}
|
||||
download()
|
||||
} finally {
|
||||
completed = true
|
||||
context.cancelCoroutines()
|
||||
context.completed = true
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
stopped = true
|
||||
context.requests.forEach { it.abort() }
|
||||
context.cancelCoroutines()
|
||||
dataTransaction.updateImage(context.imageEntity)
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (other == null || javaClass != other.javaClass) return false
|
||||
@@ -151,4 +140,9 @@ internal class ImageDownloadRunnable(
|
||||
override fun hashCode(): Int {
|
||||
return Objects.hash(imageEntity.id)
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
context.requests.forEach { it.abort() }
|
||||
context.stopped = true
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,14 @@
|
||||
package me.vripper.event
|
||||
|
||||
import kotlinx.coroutines.channels.BufferOverflow
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.asSharedFlow
|
||||
|
||||
object EventBus {
|
||||
|
||||
private val _events = MutableSharedFlow<Any>(0, Int.MAX_VALUE, BufferOverflow.DROP_OLDEST)
|
||||
private val _events = MutableSharedFlow<Any>()
|
||||
val events = _events.asSharedFlow()
|
||||
|
||||
fun publishEvent(event: Any) {
|
||||
_events.tryEmit(event)
|
||||
suspend fun publishEvent(event: Any) {
|
||||
_events.emit(event)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,7 @@ package me.vripper.host
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.*
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
@@ -48,7 +46,7 @@ internal class AcidimgHost(
|
||||
)
|
||||
)
|
||||
}.also { context.requests.add(it) }
|
||||
log.debug(String.format("Requesting %s", httpPost.uri))
|
||||
log.debug(String.format("Requesting %s", httpPost))
|
||||
val doc = try {
|
||||
httpService.client.execute(
|
||||
httpPost, context.httpContext
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
package me.vripper.host
|
||||
|
||||
import kotlinx.coroutines.cancelAndJoin
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.HostException
|
||||
@@ -19,9 +15,10 @@ import org.apache.hc.core5.http.ClassicHttpResponse
|
||||
import org.apache.hc.core5.http.Header
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import org.w3c.dom.Document
|
||||
import java.io.BufferedOutputStream
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Duration
|
||||
import java.time.LocalDateTime
|
||||
import kotlin.Throws
|
||||
|
||||
internal abstract class Host(
|
||||
@@ -103,10 +100,10 @@ internal abstract class Host(
|
||||
"vripper_",
|
||||
".tmp"
|
||||
)
|
||||
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
|
||||
return Files.newOutputStream(tempImage).use { fos ->
|
||||
val image = context.imageEntity
|
||||
synchronized(image.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val post = dataTransaction.findPostById(context.postId).orElseThrow()
|
||||
val size = if (image.size < 0) {
|
||||
response.entity.contentLength
|
||||
} else {
|
||||
@@ -127,22 +124,20 @@ internal abstract class Host(
|
||||
)
|
||||
val buffer = ByteArray(READ_BUFFER_SIZE)
|
||||
var read: Int
|
||||
val reporterJob = context.launchCoroutine {
|
||||
while (isActive) {
|
||||
dataTransaction.updateImage(image, false)
|
||||
delay(100)
|
||||
}
|
||||
}
|
||||
while (response.entity.content.read(buffer)
|
||||
.also { read = it } != -1
|
||||
var lastImageUpdateDate = LocalDateTime.now()
|
||||
while (response.entity.content.read(buffer, 0, READ_BUFFER_SIZE)
|
||||
.also { read = it } != -1 && !context.stopped
|
||||
) {
|
||||
bos.write(buffer, 0, read)
|
||||
fos.write(buffer, 0, read)
|
||||
image.downloaded += read
|
||||
if (Duration.between(lastImageUpdateDate, LocalDateTime.now()).toMillis() > 1750) {
|
||||
dataTransaction.updateImage(image)
|
||||
lastImageUpdateDate = LocalDateTime.now()
|
||||
} else {
|
||||
dataTransaction.updateImage(image, false)
|
||||
}
|
||||
downloadSpeedService.reportDownloadedBytes(read.toLong())
|
||||
}
|
||||
runBlocking {
|
||||
reporterJob.cancelAndJoin()
|
||||
}
|
||||
dataTransaction.updateImage(image)
|
||||
Pair(tempImage, mimeType)
|
||||
}
|
||||
|
||||
@@ -20,8 +20,6 @@ data class ViperSettings(
|
||||
val password: String = "",
|
||||
val thanks: Boolean = false,
|
||||
val host: String = "https://vipergirls.to",
|
||||
val requestLimit: Long = 2,
|
||||
val fetchMetadata: Boolean = false,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package me.vripper.services
|
||||
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.time.sample
|
||||
import kotlinx.serialization.json.Json
|
||||
import me.vripper.download.DownloadService
|
||||
@@ -12,9 +13,9 @@ import me.vripper.tasks.AddPostTask
|
||||
import me.vripper.tasks.ThreadLookupTask
|
||||
import me.vripper.utilities.ApplicationProperties
|
||||
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils
|
||||
import me.vripper.utilities.taskRunner
|
||||
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
|
||||
import java.sql.DriverManager
|
||||
import java.time.Duration
|
||||
@@ -42,10 +43,10 @@ internal class AppEndpointService(
|
||||
if (postLinks.isBlank()) {
|
||||
return
|
||||
}
|
||||
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isBlank() }.map { it.trim() }
|
||||
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isEmpty() }.map { it.trim() }
|
||||
.filter { it.isNotEmpty() }
|
||||
for (link in urlList) {
|
||||
log.debug("Scanning: $link")
|
||||
log.debug("Starting to process thread: $link")
|
||||
if (!link.startsWith(settingsService.settings.viperSettings.host)) {
|
||||
continue
|
||||
}
|
||||
@@ -58,17 +59,17 @@ internal class AppEndpointService(
|
||||
threadId = m.group(1).toLong()
|
||||
postId = m.group(4)?.toLong()
|
||||
if (postId == null) {
|
||||
taskRunner.submit(
|
||||
GlobalScopeCoroutine.launch {
|
||||
ThreadLookupTask(
|
||||
threadId, settingsService.settings
|
||||
)
|
||||
)
|
||||
).run()
|
||||
}
|
||||
} else {
|
||||
taskRunner.submit(
|
||||
GlobalScopeCoroutine.launch {
|
||||
AddPostTask(
|
||||
listOf(ThreadPostId(threadId, postId))
|
||||
)
|
||||
)
|
||||
).run()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("Invalid link $link, link is missing the threadId")
|
||||
@@ -80,13 +81,15 @@ internal class AppEndpointService(
|
||||
|
||||
override suspend fun restartAll(posIds: List<Long>) {
|
||||
lock.withLock {
|
||||
downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
|
||||
.map { dataTransaction.findPostByPostId(it) })
|
||||
downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
|
||||
.map { it.get() })
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun download(posts: List<ThreadPostId>) {
|
||||
taskRunner.submit(AddPostTask(posts))
|
||||
GlobalScopeCoroutine.launch {
|
||||
AddPostTask(posts).run()
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun stopAll(postIdList: List<Long>) {
|
||||
@@ -119,9 +122,6 @@ internal class AppEndpointService(
|
||||
)
|
||||
}
|
||||
val threadLookupResult = threadCacheService[thread.threadId]
|
||||
if (threadLookupResult.error.isNotBlank()) {
|
||||
throw PostParseException("Error occurred for threadId = $threadId, ${threadLookupResult.error}")
|
||||
}
|
||||
threadLookupResult.postItemList.map { postItem ->
|
||||
PostSelection(
|
||||
postItem.threadId,
|
||||
@@ -135,13 +135,23 @@ internal class AppEndpointService(
|
||||
postItem.forum,
|
||||
postItem.imageItemList.map { it.thumbLink })
|
||||
}.ifEmpty {
|
||||
log.error(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s", threadId
|
||||
)
|
||||
)
|
||||
throw PostParseException(
|
||||
"Nothing found for threadId = $threadId"
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s", threadId
|
||||
)
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
log.error(e.message)
|
||||
throw e
|
||||
throw PostParseException(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s, %s", threadId, e.message
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -175,21 +185,16 @@ internal class AppEndpointService(
|
||||
}
|
||||
|
||||
override suspend fun rename(postId: Long, newName: String) {
|
||||
taskRunner.submit {
|
||||
GlobalScopeCoroutine.launch {
|
||||
synchronized(postId.toString().intern()) {
|
||||
if (dataTransaction.exists(postId)) {
|
||||
dataTransaction.findPostByPostId(postId).let { post ->
|
||||
if (Path(post.downloadDirectory, post.folderName).exists()) {
|
||||
PathUtils.rename(
|
||||
dataTransaction.findImagesByPostId(postId),
|
||||
post.downloadDirectory,
|
||||
post.folderName,
|
||||
newName
|
||||
)
|
||||
}
|
||||
post.folderName = PathUtils.sanitize(newName)
|
||||
dataTransaction.updatePost(post)
|
||||
dataTransaction.findPostByPostId(postId).ifPresent { post ->
|
||||
if (Path(post.downloadDirectory, post.folderName).exists()) {
|
||||
PathUtils.rename(
|
||||
dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
|
||||
)
|
||||
}
|
||||
post.folderName = PathUtils.sanitize(newName)
|
||||
dataTransaction.updatePost(post)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -246,7 +251,7 @@ internal class AppEndpointService(
|
||||
}
|
||||
|
||||
override suspend fun findPost(postId: Long): Post {
|
||||
return mapper(dataTransaction.findPostByPostId(postId))
|
||||
return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
|
||||
}
|
||||
|
||||
override suspend fun findImagesByPostId(postId: Long): List<Image> {
|
||||
@@ -348,7 +353,7 @@ internal class AppEndpointService(
|
||||
val addedAt = it.getTimestamp("ADDED_AT")
|
||||
val folderName = it.getString("FOLDER_NAME") ?: ""
|
||||
|
||||
val exists = dataTransaction.exists(postId)
|
||||
val exists = dataTransaction.findPostByPostId(postId).isPresent
|
||||
if (exists) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package me.vripper.services
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.data.repositories.ImageRepository
|
||||
import me.vripper.data.repositories.MetadataRepository
|
||||
import me.vripper.data.repositories.PostRepository
|
||||
import me.vripper.data.repositories.PostDownloadStateRepository
|
||||
import me.vripper.data.repositories.ThreadRepository
|
||||
import me.vripper.entities.*
|
||||
import me.vripper.event.*
|
||||
@@ -14,73 +16,70 @@ import me.vripper.utilities.PathUtils.sanitize
|
||||
import me.vripper.vgapi.PostItem
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import java.util.*
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
internal class DataTransaction(
|
||||
private val settingsService: SettingsService,
|
||||
private val postRepository: PostRepository,
|
||||
private val postDownloadStateRepository: PostDownloadStateRepository,
|
||||
private val imageRepository: ImageRepository,
|
||||
private val threadRepository: ThreadRepository,
|
||||
private val metadataRepository: MetadataRepository,
|
||||
private val eventBus: EventBus,
|
||||
) {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
|
||||
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
|
||||
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
|
||||
transaction { postRepository.findById(id) }
|
||||
}
|
||||
|
||||
private val postPostIdCache: LoadingCache<Long, PostEntity> =
|
||||
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
|
||||
transaction { postRepository.findByPostId(id) }
|
||||
}
|
||||
|
||||
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
|
||||
return transaction { postRepository.save(postEntities) }
|
||||
return transaction { postDownloadStateRepository.save(postEntities) }
|
||||
}
|
||||
|
||||
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
|
||||
val savedPost = transaction {
|
||||
val savedPost =
|
||||
postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
|
||||
postDownloadStateRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
|
||||
save(images.map { it.copy(postIdRef = savedPost.id) })
|
||||
savedPost
|
||||
}
|
||||
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
|
||||
}
|
||||
}
|
||||
|
||||
fun updatePosts(postEntities: List<PostEntity>) {
|
||||
transaction { postRepository.update(postEntities) }
|
||||
postEntities.forEach { postEntity ->
|
||||
postPostIdCache.put(postEntity.postId, postEntity)
|
||||
postEntityIdCache.put(postEntity.id, postEntity)
|
||||
transaction { postDownloadStateRepository.update(postEntities) }
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(PostUpdateEvent(postEntities))
|
||||
}
|
||||
eventBus.publishEvent(PostUpdateEvent(postEntities))
|
||||
}
|
||||
|
||||
fun updatePost(postEntity: PostEntity) {
|
||||
transaction { postRepository.update(postEntity) }
|
||||
postPostIdCache.put(postEntity.postId, postEntity)
|
||||
postEntityIdCache.put(postEntity.id, postEntity)
|
||||
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
|
||||
transaction { postDownloadStateRepository.update(postEntity) }
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
|
||||
}
|
||||
}
|
||||
|
||||
fun save(threadEntity: ThreadEntity) {
|
||||
val savedThread = transaction { threadRepository.save(threadEntity) }
|
||||
eventBus.publishEvent(ThreadCreateEvent(savedThread))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ThreadCreateEvent(savedThread))
|
||||
}
|
||||
}
|
||||
|
||||
fun update(threadEntity: ThreadEntity) {
|
||||
transaction { threadRepository.update(threadEntity) }
|
||||
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
|
||||
}
|
||||
}
|
||||
|
||||
fun updateImages(imageEntities: List<ImageEntity>) {
|
||||
transaction { imageRepository.update(imageEntities) }
|
||||
eventBus.publishEvent(ImageEvent(imageEntities))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ImageEvent(imageEntities))
|
||||
}
|
||||
}
|
||||
|
||||
fun updateImage(imageEntity: ImageEntity, persist: Boolean = true) {
|
||||
@@ -89,14 +88,13 @@ internal class DataTransaction(
|
||||
imageRepository.update(imageEntity)
|
||||
}
|
||||
}
|
||||
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
|
||||
}
|
||||
}
|
||||
|
||||
fun exists(postId: Long): Boolean {
|
||||
if (postPostIdCache.getIfPresent(postId) != null) {
|
||||
return true
|
||||
}
|
||||
return transaction { postRepository.existByPostId(postId) }
|
||||
return transaction { postDownloadStateRepository.existByPostId(postId) }
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
@@ -144,12 +142,14 @@ internal class DataTransaction(
|
||||
}
|
||||
savedPosts
|
||||
}
|
||||
eventBus.publishEvent(PostCreateEvent(savedPosts))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(PostCreateEvent(savedPosts))
|
||||
}
|
||||
return savedPosts
|
||||
}
|
||||
|
||||
private fun getQueuePosition(): Int? {
|
||||
return postRepository.findMaxRank()
|
||||
return postDownloadStateRepository.findMaxRank()
|
||||
}
|
||||
|
||||
private fun save(imageEntities: List<ImageEntity>) {
|
||||
@@ -157,7 +157,7 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun finishPost(postId: Long, automatic: Boolean = false) {
|
||||
val post = findPostByPostId(postId)
|
||||
val post = findPostByPostId(postId).orElseThrow()
|
||||
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
|
||||
if (imagesInErrorStatus.isNotEmpty()) {
|
||||
post.status = Status.ERROR
|
||||
@@ -188,24 +188,27 @@ internal class DataTransaction(
|
||||
transaction {
|
||||
metadataRepository.deleteAllByPostId(postIds)
|
||||
imageRepository.deleteAllByPostId(postIds)
|
||||
postRepository.deleteAll(postIds)
|
||||
postDownloadStateRepository.deleteAll(postIds)
|
||||
sortPostsByRank()
|
||||
}
|
||||
postIds.forEach { postId ->
|
||||
postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
|
||||
postPostIdCache.invalidate(postId)
|
||||
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
|
||||
}
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
|
||||
}
|
||||
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
|
||||
}
|
||||
|
||||
fun removeThread(threadId: Long) {
|
||||
transaction { threadRepository.deleteByThreadId(threadId) }
|
||||
eventBus.publishEvent(ThreadDeleteEvent(threadId))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ThreadDeleteEvent(threadId))
|
||||
}
|
||||
}
|
||||
|
||||
fun clearCompleted(): List<Long> {
|
||||
val completed = transaction { postRepository.findCompleted() }
|
||||
val completed = transaction { postDownloadStateRepository.findCompleted() }
|
||||
remove(completed)
|
||||
return completed
|
||||
}
|
||||
@@ -228,12 +231,16 @@ internal class DataTransaction(
|
||||
|
||||
fun saveMetadata(metadataEntity: MetadataEntity) {
|
||||
transaction { metadataRepository.save(metadataEntity) }
|
||||
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
|
||||
}
|
||||
}
|
||||
|
||||
fun clearQueueLinks() {
|
||||
transaction { threadRepository.deleteAll() }
|
||||
eventBus.publishEvent(ThreadClearEvent())
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(ThreadClearEvent())
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
@@ -250,15 +257,15 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun setDownloadingToStopped() {
|
||||
transaction { postRepository.setDownloadingToStopped() }
|
||||
transaction { postDownloadStateRepository.setDownloadingToStopped() }
|
||||
}
|
||||
|
||||
fun findAllPosts(): List<PostEntity> {
|
||||
return transaction { postRepository.findAll() }
|
||||
return transaction { postDownloadStateRepository.findAll() }
|
||||
}
|
||||
|
||||
fun findPostById(id: Long): PostEntity {
|
||||
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
|
||||
fun findPostById(id: Long): Optional<PostEntity> {
|
||||
return transaction { postDownloadStateRepository.findById(id) }
|
||||
}
|
||||
|
||||
fun findImagesByPostId(postId: Long): List<ImageEntity> {
|
||||
@@ -287,8 +294,8 @@ internal class DataTransaction(
|
||||
return transaction { imageRepository.countError() }
|
||||
}
|
||||
|
||||
fun findPostByPostId(postId: Long): PostEntity {
|
||||
return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
|
||||
fun findPostByPostId(postId: Long): Optional<PostEntity> {
|
||||
return transaction { postDownloadStateRepository.findByPostId(postId) }
|
||||
}
|
||||
|
||||
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
|
||||
@@ -296,7 +303,7 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun findAllNonCompletedPostIds(): List<Long> {
|
||||
return transaction { postRepository.findAllNonCompletedPostIds() }
|
||||
return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
|
||||
}
|
||||
|
||||
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
|
||||
|
||||
@@ -30,6 +30,7 @@ internal class DownloadSpeedService(
|
||||
while (isActive) {
|
||||
delay(DOWNLOAD_POLL_RATE.toLong())
|
||||
val newValue = bytesCount.getAndSet(0)
|
||||
|
||||
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
|
||||
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
|
||||
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
|
||||
import org.apache.hc.core5.pool.PoolReusePolicy
|
||||
import org.apache.hc.core5.util.TimeValue
|
||||
import org.apache.hc.core5.util.Timeout
|
||||
|
||||
internal class HTTPService(
|
||||
@@ -40,10 +41,8 @@ internal class HTTPService(
|
||||
buildConnectionPool()
|
||||
buildClientBuilder()
|
||||
coroutineScope.launch {
|
||||
while (isActive) {
|
||||
pcm.closeExpired()
|
||||
delay(60_000)
|
||||
}
|
||||
pcm.closeIdle(TimeValue.ofSeconds(60))
|
||||
delay(15000)
|
||||
}
|
||||
coroutineScope.launch {
|
||||
eventBus
|
||||
@@ -65,8 +64,8 @@ internal class HTTPService(
|
||||
|
||||
private fun buildConnectionPool() {
|
||||
pcm = PoolingHttpClientConnectionManagerBuilder.create()
|
||||
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
|
||||
.setConnPoolPolicy(PoolReusePolicy.FIFO)
|
||||
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
|
||||
.setConnPoolPolicy(PoolReusePolicy.LIFO)
|
||||
.setDefaultConnectionConfig(cc)
|
||||
.setMaxConnTotal(Int.MAX_VALUE)
|
||||
.setMaxConnPerRoute(Int.MAX_VALUE)
|
||||
@@ -84,6 +83,7 @@ internal class HTTPService(
|
||||
cc = ConnectionConfig.custom()
|
||||
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
|
||||
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
|
||||
.setTimeToLive(TimeValue.ofMinutes(10))
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,29 +1,111 @@
|
||||
package me.vripper.services
|
||||
|
||||
import me.vripper.tasks.FetchMetadataTask
|
||||
import me.vripper.utilities.taskRunner
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import me.vripper.entities.MetadataEntity
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.VripperException
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.net.URIBuilder
|
||||
import org.w3c.dom.Node
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.stream.Collectors
|
||||
|
||||
internal class MetadataService(
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val httpService: HTTPService,
|
||||
private val settingsService: SettingsService,
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val vgAuthService: VGAuthService,
|
||||
) {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
|
||||
|
||||
fun init() {
|
||||
if (!settingsService.settings.viperSettings.fetchMetadata) {
|
||||
return
|
||||
}
|
||||
dataTransaction.findAllPosts().filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }
|
||||
.map { it.postId }.forEach(::fetchMetadata)
|
||||
}
|
||||
|
||||
fun fetchMetadata(postId: Long) {
|
||||
if (!settingsService.settings.viperSettings.fetchMetadata) {
|
||||
Tasks.increment()
|
||||
coroutineScope.launch {
|
||||
try {
|
||||
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
|
||||
it.setParameter(
|
||||
"p", postId.toString()
|
||||
)
|
||||
}.build())
|
||||
|
||||
val response = RequestLimit.semaphore.withPermit {
|
||||
httpService.client.execute(httpGet, vgAuthService.context) {
|
||||
if (it.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
|
||||
}
|
||||
EntityUtils.toString(it.entity)
|
||||
}
|
||||
}
|
||||
val document = HtmlUtils.clean(response)
|
||||
val postNode: Node = XpathUtils.getAsNode(
|
||||
document,
|
||||
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
|
||||
|
||||
) ?: throw VripperException("Unable to find post #'$postId'")
|
||||
|
||||
val postedBy: String = XpathUtils.getAsNode(
|
||||
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
|
||||
)?.textContent?.trim()
|
||||
?: throw VripperException("Unable to find the poster for post #'$postId'")
|
||||
|
||||
|
||||
val node: Node = XpathUtils.getAsNode(
|
||||
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
|
||||
) ?: throw VripperException("Unable to locate post content")
|
||||
val titles = findTitleInContent(node)
|
||||
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
|
||||
dataTransaction.saveMetadata(metadataEntity)
|
||||
} finally {
|
||||
Tasks.decrement()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun findTitleInContent(node: Node): List<String> {
|
||||
val altTitle: MutableList<String> = mutableListOf()
|
||||
findTitle(node, altTitle, AtomicBoolean(true))
|
||||
return altTitle.stream().distinct().collect(Collectors.toList())
|
||||
}
|
||||
|
||||
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
|
||||
if (!keepGoing.get()) {
|
||||
return
|
||||
}
|
||||
taskRunner.submit(
|
||||
FetchMetadataTask(
|
||||
postId
|
||||
)
|
||||
)
|
||||
if (node.nodeName == "a" || node.nodeName == "img") {
|
||||
keepGoing.set(false)
|
||||
return
|
||||
}
|
||||
if (node.nodeType == Node.ELEMENT_NODE) {
|
||||
for (i in 0 until node.childNodes.length) {
|
||||
val item = node.childNodes.item(i)
|
||||
findTitle(item, altTitle, keepGoing)
|
||||
if (!keepGoing.get()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
} else if (node.nodeType == Node.TEXT_NODE) {
|
||||
val text = node.textContent.trim()
|
||||
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
|
||||
text.lowercase().contains(e.lowercase())
|
||||
}) {
|
||||
altTitle.add(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
package me.vripper.services
|
||||
|
||||
import dev.failsafe.RetryPolicy
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
@@ -9,6 +8,8 @@ import kotlinx.coroutines.launch
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.SettingsUpdateEvent
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import net.jodah.failsafe.RetryPolicy
|
||||
import net.jodah.failsafe.event.ExecutionAttemptedEvent
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
||||
internal class RetryPolicyService(
|
||||
@@ -28,10 +29,16 @@ internal class RetryPolicyService(
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
|
||||
return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt {
|
||||
log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
|
||||
}.build()
|
||||
fun <T> buildRetryPolicyForDownload(message: String): RetryPolicy<T> {
|
||||
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
|
||||
log.warn(message + "#${it.attemptCount} tries failed", it.lastFailure)
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> buildGenericRetryPolicy(message: String): RetryPolicy<T> {
|
||||
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
|
||||
log.warn(message + "#${e.attemptCount} tries failed", e.lastFailure)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,9 @@
|
||||
package me.vripper.services
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.encodeToString
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.decodeFromStream
|
||||
@@ -20,6 +24,7 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
private val configPath = VRIPPER_DIR.resolve("config.json")
|
||||
private val customProxiesPath = VRIPPER_DIR.resolve("proxies.json")
|
||||
private val proxies: MutableSet<String> = HashSet()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
private val json = Json {
|
||||
encodeDefaults = true
|
||||
@@ -35,7 +40,9 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
private fun init() {
|
||||
loadViperProxies()
|
||||
restore()
|
||||
eventBus.publishEvent(SettingsUpdateEvent(settings))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(SettingsUpdateEvent(settings))
|
||||
}
|
||||
}
|
||||
|
||||
private fun loadViperProxies() {
|
||||
@@ -93,7 +100,9 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
}
|
||||
this.settings = settings.copy(viperSettings = viperSettings)
|
||||
save()
|
||||
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
|
||||
}
|
||||
}
|
||||
|
||||
private fun restore() {
|
||||
@@ -188,11 +197,5 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
"Invalid clipboard monitoring polling rate settings, values must be >= 500"
|
||||
)
|
||||
}
|
||||
|
||||
if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 5) {
|
||||
throw ValidationException(
|
||||
"Invalid request rate limit, values must be in [1,5]"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,8 @@ package me.vripper.services
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.filterIsInstance
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.SettingsUpdateEvent
|
||||
import me.vripper.vgapi.ThreadItem
|
||||
@@ -28,7 +25,9 @@ internal class ThreadCacheService(val eventBus: EventBus) {
|
||||
|
||||
private val cache: LoadingCache<Long, ThreadItem> =
|
||||
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
|
||||
ThreadLookupAPIParser(threadId).parse()
|
||||
runBlocking {
|
||||
ThreadLookupAPIParser(threadId).parse()
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(ExecutionException::class)
|
||||
|
||||
@@ -12,13 +12,12 @@ import me.vripper.event.VGUserLoginEvent
|
||||
import me.vripper.exception.VripperException
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.tasks.LeaveThanksTask
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.taskRunner
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPost
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.cookie.Cookie
|
||||
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
|
||||
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.http.message.BasicNameValuePair
|
||||
@@ -32,25 +31,8 @@ internal class VGAuthService(
|
||||
private val log by LoggerDelegate()
|
||||
val context: HttpClientContext = HttpClientContext.create()
|
||||
var loggedUser = ""
|
||||
var authenticated = false
|
||||
val clickCookies: List<Cookie>
|
||||
get() {
|
||||
return if (authenticated) {
|
||||
val useridCookie = context.cookieStore.cookies.first { it.name.equals("vg_userid") }.let { cookie ->
|
||||
BasicClientCookie(cookie.name, cookie.value).apply {
|
||||
domain = "viper.click"
|
||||
}
|
||||
}
|
||||
val passwordCookie = context.cookieStore.cookies.first { it.name.equals("vg_password") }.let { cookie ->
|
||||
BasicClientCookie(cookie.name, cookie.value).apply {
|
||||
domain = "viper.click"
|
||||
}
|
||||
}
|
||||
listOf(useridCookie, passwordCookie)
|
||||
} else {
|
||||
emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
private var authenticated = false
|
||||
|
||||
init {
|
||||
context.cookieStore = BasicCookieStore()
|
||||
@@ -68,7 +50,10 @@ internal class VGAuthService(
|
||||
log.debug("Authentication option is disabled")
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
val username = settings.viperSettings.username
|
||||
@@ -77,7 +62,11 @@ internal class VGAuthService(
|
||||
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
val postAuth = HttpPost(settings.viperSettings.host + "/login.php?do=login").also {
|
||||
@@ -89,6 +78,10 @@ internal class VGAuthService(
|
||||
BasicNameValuePair("vb_login_md5password", password)
|
||||
)
|
||||
)
|
||||
it.addHeader("Referer", settings.viperSettings.host)
|
||||
it.addHeader(
|
||||
"Host", settings.viperSettings.host.replace("https://", "").replace("http://", "")
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -110,7 +103,11 @@ internal class VGAuthService(
|
||||
} catch (e: Exception) {
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
}
|
||||
|
||||
log.error(
|
||||
"Failed to authenticate user with " + settings.viperSettings.host, e
|
||||
)
|
||||
@@ -118,12 +115,15 @@ internal class VGAuthService(
|
||||
}
|
||||
authenticated = true
|
||||
loggedUser = username
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
}
|
||||
}
|
||||
|
||||
fun leaveThanks(postEntity: PostEntity) {
|
||||
taskRunner.submit(
|
||||
LeaveThanksTask(postEntity, authenticated, context)
|
||||
)
|
||||
GlobalScopeCoroutine.launch {
|
||||
LeaveThanksTask(postEntity, authenticated, context).run()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import me.vripper.download.DownloadService
|
||||
import me.vripper.model.ThreadPostId
|
||||
import me.vripper.services.DataTransaction
|
||||
@@ -30,14 +31,14 @@ internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponen
|
||||
}
|
||||
|
||||
val link =
|
||||
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
|
||||
"https://${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
|
||||
|
||||
val cachedThread = threadCacheService.getIfPresent(threadId)
|
||||
val threadItem = cachedThread ?:
|
||||
val threadItem = cachedThread ?: runBlocking {
|
||||
PostLookupAPIParser(
|
||||
threadId, postId
|
||||
).parse()
|
||||
|
||||
}
|
||||
|
||||
if (threadItem == null) {
|
||||
log.error("Failed to load $link")
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import me.vripper.entities.MetadataEntity
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.VripperException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.net.URIBuilder
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import org.w3c.dom.Node
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.stream.Collectors
|
||||
|
||||
internal class FetchMetadataTask(
|
||||
private val postId: Long,
|
||||
) : KoinComponent, Runnable {
|
||||
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
|
||||
private val log by LoggerDelegate()
|
||||
private val settingsService: SettingsService by inject()
|
||||
private val vgAuthService: VGAuthService by inject()
|
||||
private val httpService: HTTPService by inject()
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
|
||||
override fun run() {
|
||||
try {
|
||||
if (!settingsService.settings.viperSettings.fetchMetadata) {
|
||||
log.debug("Fetching metadata is disabled")
|
||||
return
|
||||
}
|
||||
Tasks.increment()
|
||||
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
|
||||
it.setParameter(
|
||||
"p", postId.toString()
|
||||
)
|
||||
}.build())
|
||||
|
||||
RequestLimit.getPermit(1)
|
||||
log.debug("Requesting {}", httpGet.uri)
|
||||
val response = httpService.client.execute(httpGet, vgAuthService.context) {
|
||||
if (it.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
|
||||
}
|
||||
EntityUtils.toString(it.entity)
|
||||
}
|
||||
|
||||
val document = HtmlUtils.clean(response)
|
||||
val postNode: Node = XpathUtils.getAsNode(
|
||||
document,
|
||||
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
|
||||
|
||||
) ?: throw VripperException("Unable to find post #'$postId'")
|
||||
|
||||
val postedBy: String = XpathUtils.getAsNode(
|
||||
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
|
||||
)?.textContent?.trim()
|
||||
?: throw VripperException("Unable to find the poster for post #'$postId'")
|
||||
|
||||
|
||||
val node: Node = XpathUtils.getAsNode(
|
||||
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
|
||||
) ?: throw VripperException("Unable to locate post content")
|
||||
val titles = findTitleInContent(node)
|
||||
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
|
||||
dataTransaction.saveMetadata(metadataEntity)
|
||||
} finally {
|
||||
Tasks.decrement()
|
||||
}
|
||||
}
|
||||
|
||||
private fun findTitleInContent(node: Node): List<String> {
|
||||
val altTitle: MutableList<String> = mutableListOf()
|
||||
findTitle(node, altTitle, AtomicBoolean(true))
|
||||
return altTitle.stream().distinct().collect(Collectors.toList())
|
||||
}
|
||||
|
||||
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
|
||||
if (!keepGoing.get()) {
|
||||
return
|
||||
}
|
||||
if (node.nodeName == "a" || node.nodeName == "img") {
|
||||
keepGoing.set(false)
|
||||
return
|
||||
}
|
||||
if (node.nodeType == Node.ELEMENT_NODE) {
|
||||
for (i in 0 until node.childNodes.length) {
|
||||
val item = node.childNodes.item(i)
|
||||
findTitle(item, altTitle, keepGoing)
|
||||
if (!keepGoing.get()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
} else if (node.nodeType == Node.TEXT_NODE) {
|
||||
val text = node.textContent.trim()
|
||||
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
|
||||
text.lowercase().contains(e.lowercase())
|
||||
}) {
|
||||
altTitle.add(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import me.vripper.entities.PostEntity
|
||||
import me.vripper.exception.VripperException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPost
|
||||
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
@@ -21,6 +20,7 @@ internal class LeaveThanksTask(
|
||||
private val log by LoggerDelegate()
|
||||
private val cm: HTTPService by inject()
|
||||
private val settingsService: SettingsService by inject()
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
|
||||
override fun run() {
|
||||
try {
|
||||
@@ -31,23 +31,25 @@ internal class LeaveThanksTask(
|
||||
if (!settingsService.settings.viperSettings.thanks) {
|
||||
return
|
||||
}
|
||||
val postThanks = HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
|
||||
it.entity = UrlEncodedFormEntity(
|
||||
listOf(
|
||||
BasicNameValuePair("do", "post_thanks_add"),
|
||||
BasicNameValuePair("using_ajax", "1"),
|
||||
BasicNameValuePair("p", postEntity.postId.toString()),
|
||||
BasicNameValuePair("securitytoken", postEntity.token)
|
||||
|
||||
val postThanks =
|
||||
HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
|
||||
it.entity = UrlEncodedFormEntity(
|
||||
listOf(
|
||||
BasicNameValuePair("do", "post_thanks_add"),
|
||||
BasicNameValuePair("using_ajax", "1"),
|
||||
BasicNameValuePair("p", postEntity.postId.toString()),
|
||||
BasicNameValuePair("securitytoken", postEntity.token)
|
||||
)
|
||||
)
|
||||
it.addHeader("Referer", settingsService.settings.viperSettings.host)
|
||||
it.addHeader(
|
||||
"Host",
|
||||
settingsService.settings.viperSettings.host.replace("https://", "")
|
||||
.replace("http://", "")
|
||||
)
|
||||
)
|
||||
}
|
||||
RequestLimit.getPermit(1)
|
||||
log.debug("Requesting {}", postThanks.uri)
|
||||
cm.client.execute(postThanks, context) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw VripperException("Unexpected response code '${response.code}' for $postThanks")
|
||||
}
|
||||
}
|
||||
cm.client.execute(postThanks, context) { }
|
||||
} catch (e: Exception) {
|
||||
log.error("Failed to leave a thanks for $postEntity", e)
|
||||
} finally {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.LoadingTasks
|
||||
import org.koin.core.component.KoinComponent
|
||||
@@ -9,11 +13,14 @@ internal object Tasks : KoinComponent {
|
||||
|
||||
private val eventBus: EventBus by inject()
|
||||
private var current = 0
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
@Synchronized
|
||||
fun increment() {
|
||||
if (current == 0) {
|
||||
eventBus.publishEvent(LoadingTasks(true))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(LoadingTasks(true))
|
||||
}
|
||||
}
|
||||
current += 1
|
||||
}
|
||||
@@ -22,7 +29,9 @@ internal object Tasks : KoinComponent {
|
||||
fun decrement() {
|
||||
current -= 1
|
||||
if (current == 0) {
|
||||
eventBus.publishEvent(LoadingTasks(false))
|
||||
coroutineScope.launch {
|
||||
eventBus.publishEvent(LoadingTasks(false))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.entities.ThreadEntity
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.model.ThreadPostId
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.ThreadCacheService
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.taskRunner
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
|
||||
@@ -33,14 +34,15 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
|
||||
}
|
||||
|
||||
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
|
||||
taskRunner.submit(
|
||||
GlobalScopeCoroutine.launch {
|
||||
AddPostTask(threadLookupResult.postItemList.map {
|
||||
ThreadPostId(
|
||||
it.threadId, it.postId
|
||||
)
|
||||
})
|
||||
)
|
||||
}).run()
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
dataTransaction.save(
|
||||
ThreadEntity(
|
||||
title = threadLookupResult.title,
|
||||
@@ -49,6 +51,9 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
|
||||
total = threadLookupResult.postItemList.size
|
||||
)
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
|
||||
@@ -35,7 +35,7 @@ object ApplicationProperties {
|
||||
|
||||
fun latestVersion(): String {
|
||||
val request = HttpRequest.newBuilder()
|
||||
.uri(URI.create("https://api.github.com/repos/dev-claw/vripper-project/releases/latest")).build()
|
||||
.uri(URI.create("https://api.github.com/repos/death-claw/vripper-project/releases/latest")).build()
|
||||
return HttpClient.newHttpClient().use {
|
||||
val response = it.send(request, HttpResponse.BodyHandlers.ofString())
|
||||
if (response.statusCode() / 100 != 2) {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package me.vripper.utilities
|
||||
|
||||
import kotlinx.coroutines.CoroutineExceptionHandler
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
val errorHandler = CoroutineExceptionHandler { _, exception ->
|
||||
log.error("Unexpected error", exception)
|
||||
}
|
||||
|
||||
val GlobalScopeCoroutine = CoroutineScope(SupervisorJob() + Dispatchers.IO + errorHandler)
|
||||
@@ -1,42 +1,7 @@
|
||||
package me.vripper.utilities
|
||||
|
||||
import io.github.bucket4j.Bucket
|
||||
import io.github.bucket4j.BucketConfiguration
|
||||
import io.github.bucket4j.TokensInheritanceStrategy
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.flow.collectLatest
|
||||
import kotlinx.coroutines.flow.filterIsInstance
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.SettingsUpdateEvent
|
||||
import me.vripper.services.SettingsService
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.time.Duration
|
||||
import kotlinx.coroutines.sync.Semaphore
|
||||
|
||||
internal object RequestLimit : KoinComponent {
|
||||
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val settingsService: SettingsService by inject()
|
||||
private val eventBus: EventBus by inject()
|
||||
private val bucket: Bucket = Bucket.builder().addLimit {
|
||||
it.capacity(settingsService.settings.viperSettings.requestLimit)
|
||||
.refillGreedy(settingsService.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
|
||||
}.build()
|
||||
|
||||
init {
|
||||
coroutineScope.launch {
|
||||
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collectLatest { event ->
|
||||
bucket.replaceConfiguration(BucketConfiguration.builder().addLimit {
|
||||
it.capacity(event.settings.viperSettings.requestLimit)
|
||||
.refillGreedy(event.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
|
||||
}.build(), TokensInheritanceStrategy.RESET)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getPermit(tokens: Long) {
|
||||
bucket.asBlocking().consume(tokens)
|
||||
}
|
||||
internal object RequestLimit {
|
||||
val semaphore: Semaphore = Semaphore(2)
|
||||
}
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
package me.vripper.utilities
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
val taskRunner: ExecutorService = Executors.newFixedThreadPool(6)
|
||||
val downloadRunner: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
|
||||
@@ -1,26 +1,30 @@
|
||||
package me.vripper.vgapi
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.function.CheckedSupplier
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.PostParseException
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.function.CheckedSupplier
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.net.URIBuilder
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.io.ByteArrayInputStream
|
||||
import javax.xml.parsers.SAXParserFactory
|
||||
import kotlin.Throws
|
||||
|
||||
internal class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
|
||||
private val log by LoggerDelegate()
|
||||
private val settingsService: SettingsService by inject()
|
||||
private val retryPolicyService: RetryPolicyService by inject()
|
||||
private val httpService: HTTPService by inject()
|
||||
private val vgAuthService: VGAuthService by inject()
|
||||
@@ -29,36 +33,36 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
|
||||
fun parse(): ThreadItem? {
|
||||
log.debug("Parsing post $postId")
|
||||
val httpGet =
|
||||
HttpGet(URIBuilder("https://viper.click/vr.php").also {
|
||||
HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
|
||||
it.setParameter(
|
||||
"p", postId.toString()
|
||||
)
|
||||
}.build())
|
||||
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
|
||||
log.debug("Requesting {}", httpGet)
|
||||
Tasks.increment()
|
||||
return try {
|
||||
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
log.error(
|
||||
"Failed to process thread $threadId, post $postId", it.exception
|
||||
"Failed to process thread $threadId, post $postId", it.failure
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
RequestLimit.getPermit(1)
|
||||
log.info("Requesting {}", httpGet.uri)
|
||||
httpService.client.execute(
|
||||
httpGet,
|
||||
HttpClientContext.create()
|
||||
.apply { vgAuthService.clickCookies.forEach { cookieStore.addCookie(it) } }
|
||||
) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
|
||||
}
|
||||
ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
|
||||
runBlocking {
|
||||
RequestLimit.semaphore.withPermit {
|
||||
httpService.client.execute(
|
||||
httpGet, vgAuthService.context
|
||||
) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
|
||||
}
|
||||
ByteArrayInputStream(EntityUtils.toByteArray(response.entity))
|
||||
}
|
||||
}.use {
|
||||
factory.newSAXParser()
|
||||
.parse(it, threadLookupAPIResponseHandler)
|
||||
threadLookupAPIResponseHandler.result
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
} catch (e: Exception) {
|
||||
throw PostParseException(e)
|
||||
|
||||
@@ -1,62 +1,65 @@
|
||||
package me.vripper.vgapi
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.function.CheckedSupplier
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.PostParseException
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.function.CheckedSupplier
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.net.URIBuilder
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.io.ByteArrayInputStream
|
||||
import javax.xml.parsers.SAXParserFactory
|
||||
import kotlin.Throws
|
||||
|
||||
internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
|
||||
private val log by LoggerDelegate()
|
||||
private val cm: HTTPService by inject()
|
||||
private val retryPolicyService: RetryPolicyService by inject()
|
||||
private val vgAuthService: VGAuthService by inject()
|
||||
private val settingsService: SettingsService by inject()
|
||||
|
||||
@Throws(PostParseException::class)
|
||||
fun parse(): ThreadItem {
|
||||
log.debug("Parsing thread $threadId")
|
||||
val httpGet =
|
||||
HttpGet(URIBuilder("https://viper.click/vr.php").also {
|
||||
HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/vr.php").also {
|
||||
it.setParameter(
|
||||
"t",
|
||||
threadId.toString()
|
||||
)
|
||||
}.build())
|
||||
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
|
||||
log.debug("Requesting {}", httpGet)
|
||||
Tasks.increment()
|
||||
return try {
|
||||
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
log.error(
|
||||
"Failed to process thread $threadId",
|
||||
it.exception
|
||||
it.failure
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
RequestLimit.getPermit(1)
|
||||
log.info("Requesting {}", httpGet.uri)
|
||||
cm.client.execute(
|
||||
httpGet, HttpClientContext.create().apply {
|
||||
cookieStore = BasicCookieStore()
|
||||
vgAuthService.clickCookies.forEach { cookieStore.addCookie(it) }
|
||||
}
|
||||
) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
|
||||
}
|
||||
ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
|
||||
runBlocking {
|
||||
RequestLimit.semaphore.withPermit {
|
||||
cm.client.execute(
|
||||
httpGet, vgAuthService.context
|
||||
) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
|
||||
}
|
||||
ByteArrayInputStream(EntityUtils.toByteArray(response.entity))
|
||||
}
|
||||
}.use {
|
||||
factory.newSAXParser().parse(
|
||||
it,
|
||||
threadLookupAPIResponseHandler
|
||||
|
||||
@@ -7,8 +7,6 @@ message ViperSettings {
|
||||
string password = 3;
|
||||
bool thanks = 4;
|
||||
string host = 5;
|
||||
int64 requestLimit = 6;
|
||||
bool fetchMetadata = 7;
|
||||
}
|
||||
|
||||
message DownloadSettings {
|
||||
|
||||
@@ -19,7 +19,8 @@
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<fileNamePattern>${VRIPPER_DIR}/vripper.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
|
||||
<maxFileSize>10MB</maxFileSize>
|
||||
<maxHistory>5</maxHistory>
|
||||
<maxHistory>7</maxHistory>
|
||||
<totalSizeCap>1GB</totalSizeCap>
|
||||
</rollingPolicy>
|
||||
</appender>
|
||||
|
||||
|
||||
+2
-2
@@ -160,14 +160,14 @@
|
||||
<executions>
|
||||
<execution>
|
||||
<id>compile</id>
|
||||
<phase>process-sources</phase>
|
||||
<phase>compile</phase>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<jvmTarget>21</jvmTarget>
|
||||
<jvmTarget>1.8</jvmTarget>
|
||||
<compilerPlugins>
|
||||
<plugin>kotlinx-serialization</plugin>
|
||||
</compilerPlugins>
|
||||
|
||||
@@ -31,7 +31,7 @@ class VripperGuiApplication : App(
|
||||
private val log by LoggerDelegate()
|
||||
private var initialized = false
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
|
||||
init {
|
||||
APP_INSTANCE = this
|
||||
@@ -49,8 +49,8 @@ class VripperGuiApplication : App(
|
||||
with(stage) {
|
||||
width = widgetsController.currentSettings.width
|
||||
height = widgetsController.currentSettings.height
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
minWidth = 800.0
|
||||
minHeight = 600.0
|
||||
icons.addAll(
|
||||
listOf(
|
||||
Image("icons/16x16.png"),
|
||||
|
||||
@@ -23,7 +23,8 @@ class AboutFragment : Fragment("About") {
|
||||
padding = Insets(15.0, 15.0, 15.0, 15.0)
|
||||
spacing = 15.0
|
||||
imageview("icons/64x64.png")
|
||||
vbox(spacing = 5.0) {
|
||||
vbox {
|
||||
spacing = 5.0
|
||||
text("VRipper") {
|
||||
style {
|
||||
fontWeight = FontWeight.BOLD
|
||||
@@ -31,29 +32,15 @@ class AboutFragment : Fragment("About") {
|
||||
}
|
||||
}
|
||||
text("Version ${ApplicationProperties.VERSION}")
|
||||
text("Developed by dev-claw and VRipper working group")
|
||||
hbox(spacing = 5.0) {
|
||||
hyperlink {
|
||||
imageview("icons/github-mark.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
action {
|
||||
openLink("https://github.com/dev-claw/vripper-project")
|
||||
}
|
||||
}
|
||||
hyperlink {
|
||||
imageview("icons/buymeacoffee-logo.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
action {
|
||||
openLink("https://buymeacoffee.com/devclaw")
|
||||
}
|
||||
text("Developed by death-claw and VRipper working group")
|
||||
hyperlink("Home Page") {
|
||||
action {
|
||||
openLink("https://github.com/death-claw/vripper-project")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
tab("System") {
|
||||
form {
|
||||
|
||||
@@ -6,13 +6,16 @@ import javafx.geometry.Pos
|
||||
import javafx.scene.control.TextArea
|
||||
import javafx.scene.layout.Priority
|
||||
import javafx.scene.layout.VBox
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.gui.controller.PostController
|
||||
import tornadofx.*
|
||||
|
||||
class AddLinksFragment : Fragment("Add thread links") {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val textAreaProperty = SimpleStringProperty()
|
||||
private val postController: PostController by inject()
|
||||
lateinit var input: TextArea
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package me.vripper.gui.components.fragments
|
||||
|
||||
import javafx.beans.property.SimpleBooleanProperty
|
||||
import javafx.collections.FXCollections
|
||||
import tornadofx.Fragment
|
||||
import tornadofx.listview
|
||||
import tornadofx.useCheckbox
|
||||
|
||||
class ColumnSelectionFragment : Fragment("Column Selection") {
|
||||
|
||||
val map: MutableMap<String, SimpleBooleanProperty> by param()
|
||||
|
||||
override val root = listview<String> {
|
||||
items = FXCollections.observableArrayList(map.keys)
|
||||
useCheckbox { listItem ->
|
||||
map[listItem]!!
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -9,7 +9,7 @@ import tornadofx.*
|
||||
|
||||
class ConnectionSettingsFragment : Fragment("Connection Settings") {
|
||||
private val settingsController: SettingsController by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private lateinit var connectionSettings: ConnectionSettings
|
||||
val connectionSettingsModel = ConnectionSettingsModel()
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ class DownloadSettingsFragment : Fragment("Download Settings") {
|
||||
|
||||
private val settingsController: SettingsController by inject()
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private lateinit var downloadSettings: DownloadSettings
|
||||
val downloadSettingsModel = DownloadSettingsModel()
|
||||
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
package me.vripper.gui.components.fragments
|
||||
|
||||
import atlantafx.base.theme.Styles
|
||||
import javafx.beans.property.SimpleStringProperty
|
||||
import javafx.geometry.Pos
|
||||
import javafx.scene.control.ComboBox
|
||||
import kotlinx.coroutines.*
|
||||
import me.vripper.gui.controller.PostController
|
||||
import org.kordamp.ikonli.feather.Feather
|
||||
import org.kordamp.ikonli.javafx.FontIcon
|
||||
import tornadofx.*
|
||||
|
||||
class RenameFragment : Fragment("Rename download post") {
|
||||
@@ -38,8 +35,6 @@ class RenameFragment : Fragment("Rename download post") {
|
||||
}
|
||||
}
|
||||
button("Rename") {
|
||||
graphic = FontIcon.of(Feather.EDIT)
|
||||
addClass(Styles.ACCENT)
|
||||
disableWhen(comboBox.editor.textProperty().isEmpty)
|
||||
action {
|
||||
coroutineScope.launch {
|
||||
|
||||
@@ -6,7 +6,10 @@ import javafx.scene.control.Alert
|
||||
import javafx.scene.control.TabPane
|
||||
import javafx.scene.layout.Priority
|
||||
import javafx.scene.layout.VBox
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.exception.ValidationException
|
||||
import me.vripper.gui.controller.SettingsController
|
||||
import org.kordamp.ikonli.feather.Feather
|
||||
@@ -17,7 +20,7 @@ import tornadofx.*
|
||||
class SettingsFragment : Fragment("Settings") {
|
||||
|
||||
private val settingsController: SettingsController by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val downloadSettingsFragment: DownloadSettingsFragment = find()
|
||||
private val connectionSettingsFragment: ConnectionSettingsFragment = find()
|
||||
private val viperSettingsFragment: ViperSettingsFragment = find()
|
||||
@@ -28,9 +31,8 @@ class SettingsFragment : Fragment("Settings") {
|
||||
tabpane {
|
||||
tabClosingPolicy = TabPane.TabClosingPolicy.UNAVAILABLE
|
||||
VBox.setVgrow(this, Priority.ALWAYS)
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
prefHeight = 400.0
|
||||
minWidth = 700.0
|
||||
minHeight = 400.0
|
||||
tab(downloadSettingsFragment.title) {
|
||||
add(downloadSettingsFragment)
|
||||
graphic = FontIcon.of(Feather.FOLDER)
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ import tornadofx.*
|
||||
class SystemSettingsFragment : Fragment("System Settings") {
|
||||
private val settingsController: SettingsController by inject()
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private lateinit var systemSettings: SystemSettings
|
||||
val systemSettingsModel = SystemSettingsModel()
|
||||
|
||||
|
||||
+6
-21
@@ -2,7 +2,6 @@ package me.vripper.gui.components.fragments
|
||||
|
||||
import atlantafx.base.controls.ToggleSwitch
|
||||
import javafx.collections.FXCollections
|
||||
import javafx.scene.control.Spinner
|
||||
import kotlinx.coroutines.*
|
||||
import me.vripper.gui.controller.SettingsController
|
||||
import me.vripper.gui.model.settings.ViperSettingsModel
|
||||
@@ -12,7 +11,7 @@ import tornadofx.*
|
||||
class ViperSettingsFragment : Fragment("Viper Settings") {
|
||||
private val settingsController: SettingsController by inject()
|
||||
private val proxies = FXCollections.observableArrayList<String>()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private lateinit var viperGirlsSettings: ViperSettings
|
||||
val viperSettingsModel = ViperSettingsModel()
|
||||
|
||||
@@ -26,30 +25,16 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
|
||||
viperSettingsModel.password = viperGirlsSettings.password
|
||||
viperSettingsModel.thanks = viperGirlsSettings.thanks
|
||||
viperSettingsModel.host = viperGirlsSettings.host
|
||||
viperSettingsModel.requestLimit = viperGirlsSettings.requestLimit
|
||||
viperSettingsModel.fetchMetadata = viperGirlsSettings.fetchMetadata
|
||||
proxies.addAll(settingsController.getProxies())
|
||||
}.await()
|
||||
runLater {
|
||||
with(root) {
|
||||
form {
|
||||
fieldset {
|
||||
field("Viper domain") {
|
||||
field("Select a proxy") {
|
||||
combobox(viperSettingsModel.hostProperty, proxies)
|
||||
}
|
||||
field("Rate limit (requests/s)") {
|
||||
add(Spinner<Int>(1, 5, viperGirlsSettings.requestLimit.toInt()).apply {
|
||||
viperSettingsModel.requestLimitProperty.bind(valueProperty())
|
||||
isEditable = true
|
||||
atlantafx.base.util.IntegerStringConverter.createFor(this)
|
||||
})
|
||||
}
|
||||
field("Fetch Metadata") {
|
||||
checkbox {
|
||||
bind(viperSettingsModel.fetchMetadataProperty)
|
||||
}
|
||||
}
|
||||
field("Authentication") {
|
||||
field("Enable ViperGirls Authentication") {
|
||||
add(ToggleSwitch().apply {
|
||||
isSelected = viperGirlsSettings.login
|
||||
viperSettingsModel.loginProperty.bind(selectedProperty())
|
||||
@@ -57,13 +42,13 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
|
||||
}
|
||||
fieldset {
|
||||
visibleWhen(viperSettingsModel.loginProperty)
|
||||
field("Username") {
|
||||
field("ViperGirls Username") {
|
||||
textfield(viperSettingsModel.usernameProperty)
|
||||
}
|
||||
field("Password") {
|
||||
field("ViperGirls Password") {
|
||||
passwordfield(viperSettingsModel.passwordProperty)
|
||||
}
|
||||
field("Leave likes") {
|
||||
field("Leave like") {
|
||||
checkbox {
|
||||
bind(viperSettingsModel.thanksProperty)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ class ActionBarView : View() {
|
||||
private val downloadActiveProperty = SimpleBooleanProperty(true)
|
||||
private val postController: PostController by inject()
|
||||
private val postsTableView: PostsTableView by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val grpcEndpointService: IAppEndpointService by di("remoteAppEndpointService")
|
||||
private val localEndpointService: IAppEndpointService by di("localAppEndpointService")
|
||||
private val running = SimpleIntegerProperty(0)
|
||||
@@ -60,10 +60,7 @@ class ActionBarView : View() {
|
||||
action {
|
||||
find<AddLinksFragment>().apply {
|
||||
input.clear()
|
||||
}.openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
}.openModal()
|
||||
}
|
||||
}
|
||||
separator(Orientation.VERTICAL)
|
||||
@@ -114,10 +111,7 @@ class ActionBarView : View() {
|
||||
contentDisplay = ContentDisplay.GRAPHIC_ONLY
|
||||
tooltip("Open settings menu [Ctrl+P]")
|
||||
action {
|
||||
find<SettingsFragment>().openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
find<SettingsFragment>().openModal(owner = primaryStage)
|
||||
}
|
||||
}
|
||||
separator(Orientation.VERTICAL)
|
||||
|
||||
@@ -37,7 +37,7 @@ class ImagesTableView : View("Photos") {
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val localAppEndpointService: IAppEndpointService by di("localAppEndpointService")
|
||||
private val remoteAppEndpointService: IAppEndpointService by di("remoteAppEndpointService")
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val items: ObservableList<ImageModel> = FXCollections.observableArrayList()
|
||||
private var preview: Preview? = null
|
||||
|
||||
@@ -273,9 +273,7 @@ class ImagesTableView : View("Photos") {
|
||||
fun setPostId(postId: Long?) {
|
||||
ActiveUICoroutines.images.forEach { it.cancel() }
|
||||
ActiveUICoroutines.images.clear()
|
||||
runLater {
|
||||
items.clear()
|
||||
}
|
||||
items.clear()
|
||||
if (postId == null) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ class LoadingView : View("VRipper") {
|
||||
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val grpcEndpointService: GrpcEndpointService by di("remoteAppEndpointService")
|
||||
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
override val root = borderpane {}
|
||||
|
||||
@@ -51,12 +51,10 @@ class LoadingView : View("VRipper") {
|
||||
if (!grpcEndpointService.ready()) {
|
||||
val sessionView = find<SessionFragment>()
|
||||
runLater {
|
||||
sessionView.openModal()?.apply {
|
||||
setOnCloseRequest {
|
||||
sessionView.openModal().also {
|
||||
it?.setOnCloseRequest {
|
||||
VripperGuiApplication.APP_INSTANCE.stop()
|
||||
}
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ class LogTableView : View() {
|
||||
private val localAppEndpointService: IAppEndpointService by di("localAppEndpointService")
|
||||
private val remoteAppEndpointService: IAppEndpointService by di("remoteAppEndpointService")
|
||||
private val tableView: TableView<LogModel>
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val items: ObservableList<LogModel> = FXCollections.observableArrayList()
|
||||
private var maxLogEvent = 0
|
||||
|
||||
@@ -228,8 +228,8 @@ class LogTableView : View() {
|
||||
|
||||
private fun openLog(item: LogModel) {
|
||||
find<LogMessageFragment>(mapOf(LogMessageFragment::logModel to item)).openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
minWidth = 600.0
|
||||
minHeight = 400.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import tornadofx.*
|
||||
|
||||
class MenuBarView : View() {
|
||||
private val logger by LoggerDelegate()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val downloadActiveProperty = SimpleBooleanProperty(false)
|
||||
private val postsTableView: PostsTableView by inject()
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
@@ -67,10 +67,7 @@ class MenuBarView : View() {
|
||||
action {
|
||||
find<AddLinksFragment>().apply {
|
||||
input.clear()
|
||||
}.openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
}.openModal()
|
||||
}
|
||||
}
|
||||
separator()
|
||||
@@ -116,20 +113,14 @@ class MenuBarView : View() {
|
||||
item("Settings", KeyCodeCombination(KeyCode.P, KeyCombination.CONTROL_DOWN)).apply {
|
||||
graphic = FontIcon.of(Feather.SETTINGS)
|
||||
action {
|
||||
find<SettingsFragment>().openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
find<SettingsFragment>().openModal(owner = primaryStage)
|
||||
}
|
||||
}
|
||||
separator()
|
||||
item("Change session", KeyCodeCombination(KeyCode.S, KeyCombination.SHIFT_DOWN)) {
|
||||
graphic = FontIcon.of(Feather.LINK_2)
|
||||
action {
|
||||
find<SessionFragment>().openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
find<SessionFragment>().openModal()
|
||||
}
|
||||
}
|
||||
separator()
|
||||
@@ -197,7 +188,7 @@ class MenuBarView : View() {
|
||||
owner = primaryStage,
|
||||
) {
|
||||
if (it == ButtonType.YES) {
|
||||
openLink("https://github.com/dev-claw/vripper-project/releases/tag/$latestVersion")
|
||||
openLink("https://github.com/death-claw/vripper-project/releases/tag/$latestVersion")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -215,8 +206,8 @@ class MenuBarView : View() {
|
||||
graphic = FontIcon.of(Feather.INFO)
|
||||
action {
|
||||
find<AboutFragment>().openModal()?.apply {
|
||||
this.minWidth = 100.0
|
||||
this.minHeight = 100.0
|
||||
this.minWidth = 625.0
|
||||
this.minHeight = 200.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import tornadofx.*
|
||||
class PostInfoView : View() {
|
||||
private val logger by LoggerDelegate()
|
||||
private val postController: PostController by inject()
|
||||
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val imagesTableView: ImagesTableView by inject()
|
||||
private val postModel: PostModel = PostModel(
|
||||
-1, "", 0.0, "", "", 0, 0, "", "", -1, "", "", "", emptyList(), emptyList(), "", 0
|
||||
|
||||
@@ -37,7 +37,7 @@ import kotlin.io.path.Path
|
||||
|
||||
class PostsTableView : View() {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val postController: PostController by inject()
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val clipboardService: ClipboardService by inject()
|
||||
@@ -97,7 +97,7 @@ class PostsTableView : View() {
|
||||
isTableMenuButtonVisible = true
|
||||
primaryStage.addEventFilter(KeyEvent.KEY_PRESSED) { event ->
|
||||
if (event.code == KeyCode.DELETE) {
|
||||
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty() && this.isFocused) {
|
||||
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty()) {
|
||||
deleteSelected()
|
||||
}
|
||||
}
|
||||
@@ -178,10 +178,7 @@ class PostsTableView : View() {
|
||||
action {
|
||||
find<AddLinksFragment>().apply {
|
||||
input.clear()
|
||||
}.openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
}
|
||||
}.openModal()
|
||||
}
|
||||
})
|
||||
column("Preview", PostModel::previewListProperty) {
|
||||
@@ -417,7 +414,7 @@ class PostsTableView : View() {
|
||||
coroutineScope.launch {
|
||||
postController.onNewPosts().collect {
|
||||
runLater {
|
||||
items.addAll(it)
|
||||
tableView.items.addAll(it)
|
||||
tableView.sort()
|
||||
}
|
||||
}
|
||||
@@ -446,7 +443,7 @@ class PostsTableView : View() {
|
||||
coroutineScope.launch {
|
||||
postController.onDeletePosts().collect {
|
||||
runLater {
|
||||
items.items.removeIf { p -> p.postId == it }
|
||||
items.removeIf { p -> p.postId == it }
|
||||
tableView.sort()
|
||||
}
|
||||
}
|
||||
@@ -472,8 +469,7 @@ class PostsTableView : View() {
|
||||
RenameFragment::altTitles to post.altTitles
|
||||
)
|
||||
).openModal()?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
minWidth = 450.0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -504,7 +500,7 @@ class PostsTableView : View() {
|
||||
coroutineScope.launch {
|
||||
postController.delete(postIdList)
|
||||
runLater {
|
||||
items.items.removeIf { postIdList.contains(it.postId) }
|
||||
tableView.items.removeIf { postIdList.contains(it.postId) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import tornadofx.*
|
||||
class StatusBarView : View("Status bar") {
|
||||
private val logger by LoggerDelegate()
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val grpcEndpointService: IAppEndpointService by di("remoteAppEndpointService")
|
||||
private val localEndpointService: IAppEndpointService by di("localAppEndpointService")
|
||||
private val remoteText = SimpleStringProperty()
|
||||
|
||||
@@ -24,7 +24,7 @@ import tornadofx.*
|
||||
|
||||
class ThreadTableView : View() {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private val threadController: ThreadController by inject()
|
||||
private val widgetsController: WidgetsController by inject()
|
||||
private val mainView: MainView by inject()
|
||||
@@ -65,7 +65,7 @@ class ThreadTableView : View() {
|
||||
isTableMenuButtonVisible = true
|
||||
primaryStage.addEventFilter(KeyEvent.KEY_PRESSED) { event ->
|
||||
if (event.code == KeyCode.DELETE) {
|
||||
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty() && this.isFocused) {
|
||||
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty()) {
|
||||
deleteSelected()
|
||||
}
|
||||
}
|
||||
@@ -248,8 +248,10 @@ class ThreadTableView : View() {
|
||||
private fun selectPosts(threadId: Long) {
|
||||
find<ThreadSelectionTableFragment>(mapOf(ThreadSelectionTableFragment::threadId to threadId)).openModal()
|
||||
?.apply {
|
||||
minWidth = 100.0
|
||||
minHeight = 100.0
|
||||
minWidth = 600.0
|
||||
minHeight = 400.0
|
||||
width = 800.0
|
||||
height = 600.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -59,16 +59,14 @@ class SettingsController : Controller() {
|
||||
viperSettingsModel.password,
|
||||
viperSettingsModel.thanks,
|
||||
viperSettingsModel.host,
|
||||
viperSettingsModel.requestLimit,
|
||||
viperSettingsModel.fetchMetadata,
|
||||
),
|
||||
systemSettings =
|
||||
SystemSettings(
|
||||
systemSettingsModel.tempPath,
|
||||
systemSettingsModel.enable,
|
||||
systemSettingsModel.pollingRate,
|
||||
systemSettingsModel.logEntries
|
||||
)
|
||||
SystemSettings(
|
||||
systemSettingsModel.tempPath,
|
||||
systemSettingsModel.enable,
|
||||
systemSettingsModel.pollingRate,
|
||||
systemSettingsModel.logEntries
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package me.vripper.gui.model.settings
|
||||
|
||||
import javafx.beans.property.SimpleBooleanProperty
|
||||
import javafx.beans.property.SimpleLongProperty
|
||||
import javafx.beans.property.SimpleStringProperty
|
||||
import tornadofx.getValue
|
||||
import tornadofx.setValue
|
||||
@@ -21,10 +20,4 @@ class ViperSettingsModel {
|
||||
|
||||
val hostProperty = SimpleStringProperty()
|
||||
var host: String by hostProperty
|
||||
|
||||
val requestLimitProperty = SimpleLongProperty()
|
||||
var requestLimit: Long by requestLimitProperty
|
||||
|
||||
val fetchMetadataProperty = SimpleBooleanProperty()
|
||||
var fetchMetadata: Boolean by fetchMetadataProperty
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import tornadofx.runLater
|
||||
class ClipboardService : Controller() {
|
||||
private val logger by LoggerDelegate()
|
||||
private var current: String? = null
|
||||
private var coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private var coroutineScope = CoroutineScope(SupervisorJob())
|
||||
private var pollJob: Job? = null
|
||||
private var subscribeJob: Job? = null
|
||||
|
||||
|
||||
@@ -176,8 +176,6 @@ class GrpcEndpointService : IAppEndpointService {
|
||||
password = settings.viperSettings.password
|
||||
thanks = settings.viperSettings.thanks
|
||||
host = settings.viperSettings.host
|
||||
requestLimit = settings.viperSettings.requestLimit
|
||||
fetchMetadata = settings.viperSettings.fetchMetadata
|
||||
build()
|
||||
}
|
||||
|
||||
@@ -243,7 +241,6 @@ class GrpcEndpointService : IAppEndpointService {
|
||||
password = settings.viperSettings.password,
|
||||
thanks = settings.viperSettings.thanks,
|
||||
host = settings.viperSettings.host,
|
||||
requestLimit = settings.viperSettings.requestLimit,
|
||||
),
|
||||
downloadSettings = DownloadSettings(
|
||||
downloadPath = settings.downloadSettings.downloadPath,
|
||||
|
||||
@@ -25,7 +25,7 @@ class Preview(owner: Stage, private val images: List<String>, private val cacheP
|
||||
|
||||
private var hBox: HBox = HBox().apply { spacing = 5.0; alignment = Pos.BOTTOM_CENTER }
|
||||
private val log by LoggerDelegate()
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob())
|
||||
val previewPopup = Popup()
|
||||
|
||||
private val cache: LoadingCache<String, ByteArray> = Caffeine
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 6.2 KiB |
+1
-1
@@ -106,7 +106,7 @@
|
||||
<executions>
|
||||
<execution>
|
||||
<id>compile</id>
|
||||
<phase>process-sources</phase>
|
||||
<phase>compile</phase>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
|
||||
@@ -196,8 +196,6 @@ class GrpcServerAppEndpointService : EndpointServiceGrpcKt.EndpointServiceCorout
|
||||
password = request.viperSettings.password,
|
||||
thanks = request.viperSettings.thanks,
|
||||
host = request.viperSettings.host,
|
||||
requestLimit = request.viperSettings.requestLimit,
|
||||
fetchMetadata = request.viperSettings.fetchMetadata,
|
||||
), systemSettings = SystemSettings(
|
||||
tempPath = request.systemSettings.tempPath,
|
||||
enableClipboardMonitoring = request.systemSettings.enableClipboardMonitoring,
|
||||
@@ -230,8 +228,6 @@ class GrpcServerAppEndpointService : EndpointServiceGrpcKt.EndpointServiceCorout
|
||||
password = settings.viperSettings.password
|
||||
thanks = settings.viperSettings.thanks
|
||||
host = settings.viperSettings.host
|
||||
requestLimit = settings.viperSettings.requestLimit
|
||||
fetchMetadata = settings.viperSettings.fetchMetadata
|
||||
build()
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ class DataBroadcast(
|
||||
private val template: SimpMessagingTemplate
|
||||
) : KoinComponent {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val appEndpointService: IAppEndpointService by inject(named("localAppEndpointService"))
|
||||
|
||||
@OptIn(FlowPreview::class)
|
||||
|
||||
Reference in New Issue
Block a user